appwrite-utils-cli 1.2.6 → 1.2.7

package/dist/migrations/comprehensiveTransfer.d.ts

@@ -102,9 +102,33 @@ export declare class ComprehensiveTransfer {
      */
     private createCollectionIndexesWithStatusCheck;
     /**
-     * Helper method to transfer documents between databases
+     * Helper method to transfer documents between databases using bulk operations with content and permission-based filtering
      */
     private transferDocumentsBetweenDatabases;
+    /**
+     * Fetch target documents by IDs in batches to check existence and permissions
+     */
+    private fetchTargetDocumentsBatch;
+    /**
+     * Transfer documents using bulk operations with proper batch size handling
+     */
+    private transferDocumentsBulk;
+    /**
+     * Direct HTTP implementation of bulk upsert API
+     */
+    private bulkUpsertDocuments;
+    /**
+     * Transfer documents individually with rate limiting
+     */
+    private transferDocumentsIndividual;
+    /**
+     * Update documents individually with content and/or permission changes
+     */
+    private updateDocumentsIndividual;
+    /**
+     * Utility method to chunk arrays
+     */
+    private chunkArray;
     /**
      * Helper method to fetch all teams with pagination
      */

package/dist/migrations/comprehensiveTransfer.js

@@ -1,4 +1,4 @@
-import { converterFunctions, tryAwaitWithRetry, parseAttribute } from "appwrite-utils";
+import { converterFunctions, tryAwaitWithRetry, parseAttribute, objectNeedsUpdate } from "appwrite-utils";
 import { Client, Databases, Storage, Users, Functions, Teams, Query, } from "node-appwrite";
 import { InputFile } from "node-appwrite/file";
 import { MessageFormatter } from "../shared/messageFormatter.js";
@@ -361,6 +361,27 @@ export class ComprehensiveTransfer {
                         await this.createBucketWithFallback(bucket);
                         MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
                     }
+                    else {
+                        // Compare bucket permissions and update if needed
+                        const sourcePermissions = JSON.stringify(bucket.$permissions?.sort() || []);
+                        const targetPermissions = JSON.stringify(existingBucket.$permissions?.sort() || []);
+                        if (sourcePermissions !== targetPermissions ||
+                            existingBucket.name !== bucket.name ||
+                            existingBucket.fileSecurity !== bucket.fileSecurity ||
+                            existingBucket.enabled !== bucket.enabled) {
+                            MessageFormatter.warning(`Bucket ${bucket.name} exists but has different settings. Updating to match source.`, { prefix: "Transfer" });
+                            try {
+                                await this.targetStorage.updateBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
+                                MessageFormatter.success(`Updated bucket ${bucket.name} to match source`, { prefix: "Transfer" });
+                            }
+                            catch (updateError) {
+                                MessageFormatter.error(`Failed to update bucket ${bucket.name}`, updateError instanceof Error ? updateError : new Error(String(updateError)), { prefix: "Transfer" });
+                            }
+                        }
+                        else {
+                            MessageFormatter.info(`Bucket ${bucket.name} already exists with matching settings`, { prefix: "Transfer" });
+                        }
+                    }
                     // Transfer bucket files with enhanced validation
                     await this.transferBucketFiles(bucket.$id, bucket.$id);
                     this.results.buckets.transferred++;
@@ -481,10 +502,27 @@ export class ComprehensiveTransfer {
             // Process files with rate limiting
             const fileTasks = files.files.map(file => this.fileLimit(async () => {
                 try {
-                    // Check if file already exists
+                    // Check if file already exists and compare permissions
+                    let existingFile = null;
                     try {
-                        await this.targetStorage.getFile(targetBucketId, file.$id);
-                        MessageFormatter.info(`File ${file.name} already exists, skipping`, { prefix: "Transfer" });
+                        existingFile = await this.targetStorage.getFile(targetBucketId, file.$id);
+                        // Compare permissions between source and target file
+                        const sourcePermissions = JSON.stringify(file.$permissions?.sort() || []);
+                        const targetPermissions = JSON.stringify(existingFile.$permissions?.sort() || []);
+                        if (sourcePermissions !== targetPermissions) {
+                            MessageFormatter.warning(`File ${file.name} (${file.$id}) exists but has different permissions. Source: ${sourcePermissions}, Target: ${targetPermissions}`, { prefix: "Transfer" });
+                            // Update file permissions to match source
+                            try {
+                                await this.targetStorage.updateFile(targetBucketId, file.$id, file.name, file.$permissions);
+                                MessageFormatter.success(`Updated file ${file.name} permissions to match source`, { prefix: "Transfer" });
+                            }
+                            catch (updateError) {
+                                MessageFormatter.error(`Failed to update permissions for file ${file.name}`, updateError instanceof Error ? updateError : new Error(String(updateError)), { prefix: "Transfer" });
+                            }
+                        }
+                        else {
+                            MessageFormatter.info(`File ${file.name} already exists with matching permissions, skipping`, { prefix: "Transfer" });
+                        }
                         return;
                     }
                     catch (error) {
@@ -702,50 +740,259 @@ export class ComprehensiveTransfer {
         return await createOrUpdateIndexesWithStatusCheck(dbId, databases, collectionId, collection, indexes);
     }
     /**
-     * Helper method to transfer documents between databases
+     * Helper method to transfer documents between databases using bulk operations with content and permission-based filtering
      */
     async transferDocumentsBetweenDatabases(sourceDb, targetDb, sourceDbId, targetDbId, sourceCollectionId, targetCollectionId) {
-        MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+        MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId} with bulk operations, content comparison, and permission filtering`, { prefix: "Transfer" });
         let lastId;
         let totalTransferred = 0;
+        let totalSkipped = 0;
+        let totalUpdated = 0;
+        // Check if bulk operations are supported
+        const supportsBulk = this.options.sourceEndpoint.includes('cloud.appwrite.io') ||
+            this.options.targetEndpoint.includes('cloud.appwrite.io');
+        if (supportsBulk) {
+            MessageFormatter.info(`Using bulk operations for enhanced performance`, { prefix: "Transfer" });
+        }
         while (true) {
-            const queries = [Query.limit(50)]; // Smaller batch size for better performance
+            // Fetch source documents in larger batches (1000 instead of 50)
+            const queries = [Query.limit(1000)];
             if (lastId) {
                 queries.push(Query.cursorAfter(lastId));
             }
-            const documents = await tryAwaitWithRetry(async () => sourceDb.listDocuments(sourceDbId, sourceCollectionId, queries));
-            if (documents.documents.length === 0) {
+            const sourceDocuments = await tryAwaitWithRetry(async () => sourceDb.listDocuments(sourceDbId, sourceCollectionId, queries));
+            if (sourceDocuments.documents.length === 0) {
                 break;
             }
-            // Transfer documents with rate limiting
-            const transferTasks = documents.documents.map(doc => this.limit(async () => {
-                try {
-                    // Check if document already exists
-                    try {
-                        await targetDb.getDocument(targetDbId, targetCollectionId, doc.$id);
-                        MessageFormatter.info(`Document ${doc.$id} already exists, skipping`, { prefix: "Transfer" });
-                        return;
+            MessageFormatter.info(`Processing batch of ${sourceDocuments.documents.length} source documents`, { prefix: "Transfer" });
+            // Extract document IDs from the current batch
+            const sourceDocIds = sourceDocuments.documents.map(doc => doc.$id);
+            // Fetch existing documents from target in a single query
+            const existingTargetDocs = await this.fetchTargetDocumentsBatch(targetDb, targetDbId, targetCollectionId, sourceDocIds);
+            // Create a map for quick lookup of existing documents
+            const existingDocsMap = new Map();
+            existingTargetDocs.forEach(doc => {
+                existingDocsMap.set(doc.$id, doc);
+            });
+            // Filter documents based on existence, content comparison, and permission comparison
+            const documentsToTransfer = [];
+            const documentsToUpdate = [];
+            for (const sourceDoc of sourceDocuments.documents) {
+                const existingTargetDoc = existingDocsMap.get(sourceDoc.$id);
+                if (!existingTargetDoc) {
+                    // Document doesn't exist in target, needs to be transferred
+                    documentsToTransfer.push(sourceDoc);
+                }
+                else {
+                    // Document exists, compare both content and permissions
+                    const sourcePermissions = JSON.stringify((sourceDoc.$permissions || []).sort());
+                    const targetPermissions = JSON.stringify((existingTargetDoc.$permissions || []).sort());
+                    const permissionsDiffer = sourcePermissions !== targetPermissions;
+                    // Use objectNeedsUpdate to compare document content (excluding system fields)
+                    const contentDiffers = objectNeedsUpdate(existingTargetDoc, sourceDoc);
+                    if (contentDiffers && permissionsDiffer) {
+                        // Both content and permissions differ
+                        documentsToUpdate.push({
+                            doc: sourceDoc,
+                            targetDoc: existingTargetDoc,
+                            reason: "content and permissions differ"
+                        });
+                        MessageFormatter.info(`Document ${sourceDoc.$id} exists but content and permissions differ - will update`, { prefix: "Transfer" });
                     }
-                    catch (error) {
-                        // Document doesn't exist, proceed with creation
+                    else if (contentDiffers) {
+                        // Only content differs
+                        documentsToUpdate.push({
+                            doc: sourceDoc,
+                            targetDoc: existingTargetDoc,
+                            reason: "content differs"
+                        });
+                        MessageFormatter.info(`Document ${sourceDoc.$id} exists but content differs - will update`, { prefix: "Transfer" });
+                    }
+                    else if (permissionsDiffer) {
+                        // Only permissions differ
+                        documentsToUpdate.push({
+                            doc: sourceDoc,
+                            targetDoc: existingTargetDoc,
+                            reason: "permissions differ"
+                        });
+                        MessageFormatter.info(`Document ${sourceDoc.$id} exists but permissions differ - will update`, { prefix: "Transfer" });
+                    }
+                    else {
+                        // Document exists with identical content AND permissions, skip
+                        totalSkipped++;
+                        MessageFormatter.info(`Document ${sourceDoc.$id} exists with matching content and permissions - skipping`, { prefix: "Transfer" });
                     }
-                    // Create document in target
-                    const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
-                    await tryAwaitWithRetry(async () => targetDb.createDocument(targetDbId, targetCollectionId, doc.$id, docData, doc.$permissions));
-                    totalTransferred++;
-                    MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
                 }
-                catch (error) {
-                    MessageFormatter.error(`Failed to transfer document ${doc.$id}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+            }
+            MessageFormatter.info(`Batch analysis: ${documentsToTransfer.length} to create, ${documentsToUpdate.length} to update, ${totalSkipped} skipped so far`, { prefix: "Transfer" });
+            // Process new documents with bulk operations if supported and available
+            if (documentsToTransfer.length > 0) {
+                if (supportsBulk && documentsToTransfer.length >= 10) {
+                    // Use bulk operations for large batches
+                    await this.transferDocumentsBulk(targetDb, targetDbId, targetCollectionId, documentsToTransfer);
+                    totalTransferred += documentsToTransfer.length;
+                    MessageFormatter.success(`Bulk transferred ${documentsToTransfer.length} new documents`, { prefix: "Transfer" });
                 }
-            }));
-            await Promise.all(transferTasks);
-            if (documents.documents.length < 50) {
+                else {
+                    // Use individual transfers for smaller batches or non-bulk endpoints
+                    const transferCount = await this.transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documentsToTransfer);
+                    totalTransferred += transferCount;
+                }
+            }
+            // Process document updates (always individual since bulk update with permissions needs special handling)
+            if (documentsToUpdate.length > 0) {
+                const updateCount = await this.updateDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documentsToUpdate);
+                totalUpdated += updateCount;
+            }
+            if (sourceDocuments.documents.length < 1000) {
                 break;
             }
-            lastId = documents.documents[documents.documents.length - 1].$id;
+            lastId = sourceDocuments.documents[sourceDocuments.documents.length - 1].$id;
+        }
+        MessageFormatter.info(`Transfer complete: ${totalTransferred} new, ${totalUpdated} updated, ${totalSkipped} skipped from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+    }
+    /**
+     * Fetch target documents by IDs in batches to check existence and permissions
+     */
+    async fetchTargetDocumentsBatch(targetDb, targetDbId, targetCollectionId, docIds) {
+        const documents = [];
+        // Split IDs into chunks of 100 for Query.equal limitations
+        const idChunks = this.chunkArray(docIds, 100);
+        for (const chunk of idChunks) {
+            try {
+                const result = await tryAwaitWithRetry(async () => targetDb.listDocuments(targetDbId, targetCollectionId, [
+                    Query.equal('$id', chunk),
+                    Query.limit(100)
+                ]));
+                documents.push(...result.documents);
+            }
+            catch (error) {
+                // If query fails, fall back to individual gets (less efficient but more reliable)
+                MessageFormatter.warning(`Batch query failed for ${chunk.length} documents, falling back to individual checks`, { prefix: "Transfer" });
+                for (const docId of chunk) {
+                    try {
+                        const doc = await targetDb.getDocument(targetDbId, targetCollectionId, docId);
+                        documents.push(doc);
+                    }
+                    catch (getError) {
+                        // Document doesn't exist, which is fine
+                    }
+                }
+            }
+        }
+        return documents;
+    }
+    /**
+     * Transfer documents using bulk operations with proper batch size handling
+     */
+    async transferDocumentsBulk(targetDb, targetDbId, targetCollectionId, documents) {
+        // Prepare documents for bulk upsert
+        const preparedDocs = documents.map(doc => {
+            const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+            return {
+                $id,
+                $permissions,
+                ...docData
+            };
+        });
+        // Process in smaller chunks for bulk operations (1000 for Pro, 100 for Free tier)
+        const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
+        let processed = false;
+        for (const maxBatchSize of batchSizes) {
+            const documentBatches = this.chunkArray(preparedDocs, maxBatchSize);
+            try {
+                for (const batch of documentBatches) {
+                    MessageFormatter.info(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
+                    await this.bulkUpsertDocuments(this.targetClient, targetDbId, targetCollectionId, batch);
+                    MessageFormatter.success(`✅ Bulk upserted ${batch.length} documents`, { prefix: "Transfer" });
+                    // Add delay between batches to respect rate limits
+                    if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+                        await new Promise(resolve => setTimeout(resolve, 200));
+                    }
+                }
+                processed = true;
+                break; // Success, exit batch size loop
+            }
+            catch (error) {
+                MessageFormatter.warning(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`, { prefix: "Transfer" });
+                continue; // Try next smaller batch size
+            }
+        }
+        if (!processed) {
+            MessageFormatter.warning(`All bulk operations failed, falling back to individual transfers`, { prefix: "Transfer" });
+            // Fall back to individual transfers
+            await this.transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documents);
+        }
+    }
+    /**
+     * Direct HTTP implementation of bulk upsert API
+     */
+    async bulkUpsertDocuments(client, dbId, collectionId, documents) {
+        const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+        const url = new URL(client.config.endpoint + apiPath);
+        const headers = {
+            'Content-Type': 'application/json',
+            'X-Appwrite-Project': client.config.project,
+            'X-Appwrite-Key': client.config.key
+        };
+        const response = await fetch(url.toString(), {
+            method: 'PUT',
+            headers,
+            body: JSON.stringify({ documents })
+        });
+        if (!response.ok) {
+            const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
+            throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+        }
+        return await response.json();
+    }
+    /**
+     * Transfer documents individually with rate limiting
+     */
+    async transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documents) {
+        let successCount = 0;
+        const transferTasks = documents.map(doc => this.limit(async () => {
+            try {
+                const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+                await tryAwaitWithRetry(async () => targetDb.createDocument(targetDbId, targetCollectionId, doc.$id, docData, doc.$permissions));
+                successCount++;
+                MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
+            }
+            catch (error) {
+                MessageFormatter.error(`Failed to transfer document ${doc.$id}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+            }
+        }));
+        await Promise.all(transferTasks);
+        return successCount;
+    }
+    /**
+     * Update documents individually with content and/or permission changes
+     */
+    async updateDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documentPairs) {
+        let successCount = 0;
+        const updateTasks = documentPairs.map(({ doc, targetDoc, reason }) => this.limit(async () => {
+            try {
+                const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+                await tryAwaitWithRetry(async () => targetDb.updateDocument(targetDbId, targetCollectionId, doc.$id, docData, doc.$permissions));
+                successCount++;
+                MessageFormatter.success(`Updated document ${doc.$id} (${reason}) - permissions: [${targetDoc.$permissions?.join(', ')}] → [${doc.$permissions?.join(', ')}]`, { prefix: "Transfer" });
+            }
+            catch (error) {
+                MessageFormatter.error(`Failed to update document ${doc.$id} (${reason})`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+            }
+        }));
+        await Promise.all(updateTasks);
+        return successCount;
+    }
+    /**
+     * Utility method to chunk arrays
+     */
+    chunkArray(array, size) {
+        const chunks = [];
+        for (let i = 0; i < array.length; i += size) {
+            chunks.push(array.slice(i, i + size));
         }
-        MessageFormatter.info(`Transferred ${totalTransferred} documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+        return chunks;
     }
     /**
      * Helper method to fetch all teams with pagination
@@ -810,10 +1057,27 @@ export class ComprehensiveTransfer {
             // Transfer memberships with rate limiting
             const transferTasks = memberships.map(membership => this.userLimit(async () => {
                 try {
-                    // Check if membership already exists
+                    // Check if membership already exists and compare roles
+                    let existingMembership = null;
                     try {
-                        await this.targetTeams.getMembership(teamId, membership.$id);
-                        MessageFormatter.info(`Membership ${membership.$id} already exists, skipping`, { prefix: "Transfer" });
+                        existingMembership = await this.targetTeams.getMembership(teamId, membership.$id);
+                        // Compare roles between source and target membership
+                        const sourceRoles = JSON.stringify(membership.roles?.sort() || []);
+                        const targetRoles = JSON.stringify(existingMembership.roles?.sort() || []);
+                        if (sourceRoles !== targetRoles) {
+                            MessageFormatter.warning(`Membership ${membership.$id} exists but has different roles. Source: ${sourceRoles}, Target: ${targetRoles}`, { prefix: "Transfer" });
+                            // Update membership roles to match source
+                            try {
+                                await this.targetTeams.updateMembership(teamId, membership.$id, membership.roles);
+                                MessageFormatter.success(`Updated membership ${membership.$id} roles to match source`, { prefix: "Transfer" });
+                            }
+                            catch (updateError) {
+                                MessageFormatter.error(`Failed to update roles for membership ${membership.$id}`, updateError instanceof Error ? updateError : new Error(String(updateError)), { prefix: "Transfer" });
+                            }
+                        }
+                        else {
+                            MessageFormatter.info(`Membership ${membership.$id} already exists with matching roles, skipping`, { prefix: "Transfer" });
+                        }
                         return;
                     }
                     catch (error) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.2.6",
+  "version": "1.2.7",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/comprehensiveTransfer.ts

@@ -1,4 +1,4 @@
-import { converterFunctions, tryAwaitWithRetry, parseAttribute } from "appwrite-utils";
+import { converterFunctions, tryAwaitWithRetry, parseAttribute, objectNeedsUpdate } from "appwrite-utils";
 import {
   Client,
   Databases,
@@ -519,6 +519,45 @@ export class ComprehensiveTransfer {
               // Create bucket with fallback strategy for maximumFileSize
               await this.createBucketWithFallback(bucket);
               MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
+            } else {
+              // Compare bucket permissions and update if needed
+              const sourcePermissions = JSON.stringify(bucket.$permissions?.sort() || []);
+              const targetPermissions = JSON.stringify(existingBucket.$permissions?.sort() || []);
+              
+              if (sourcePermissions !== targetPermissions ||
+                  existingBucket.name !== bucket.name ||
+                  existingBucket.fileSecurity !== bucket.fileSecurity ||
+                  existingBucket.enabled !== bucket.enabled) {
+                
+                MessageFormatter.warning(
+                  `Bucket ${bucket.name} exists but has different settings. Updating to match source.`, 
+                  { prefix: "Transfer" }
+                );
+                
+                try {
+                  await this.targetStorage.updateBucket(
+                    bucket.$id,
+                    bucket.name,
+                    bucket.$permissions,
+                    bucket.fileSecurity,
+                    bucket.enabled,
+                    bucket.maximumFileSize,
+                    bucket.allowedFileExtensions,
+                    bucket.compression as any,
+                    bucket.encryption,
+                    bucket.antivirus
+                  );
+                  MessageFormatter.success(`Updated bucket ${bucket.name} to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update bucket ${bucket.name}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`Bucket ${bucket.name} already exists with matching settings`, { prefix: "Transfer" });
+              }
             }
 
             // Transfer bucket files with enhanced validation
@@ -703,10 +742,40 @@ export class ComprehensiveTransfer {
       const fileTasks = files.files.map(file => 
         this.fileLimit(async () => {
           try {
-            // Check if file already exists
+            // Check if file already exists and compare permissions
+            let existingFile: Models.File | null = null;
             try {
-              await this.targetStorage.getFile(targetBucketId, file.$id);
-              MessageFormatter.info(`File ${file.name} already exists, skipping`, { prefix: "Transfer" });
+              existingFile = await this.targetStorage.getFile(targetBucketId, file.$id);
+              
+              // Compare permissions between source and target file
+              const sourcePermissions = JSON.stringify(file.$permissions?.sort() || []);
+              const targetPermissions = JSON.stringify(existingFile.$permissions?.sort() || []);
+              
+              if (sourcePermissions !== targetPermissions) {
+                MessageFormatter.warning(
+                  `File ${file.name} (${file.$id}) exists but has different permissions. Source: ${sourcePermissions}, Target: ${targetPermissions}`, 
+                  { prefix: "Transfer" }
+                );
+                
+                // Update file permissions to match source
+                try {
+                  await this.targetStorage.updateFile(
+                    targetBucketId,
+                    file.$id,
+                    file.name,
+                    file.$permissions
+                  );
+                  MessageFormatter.success(`Updated file ${file.name} permissions to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update permissions for file ${file.name}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`File ${file.name} already exists with matching permissions, skipping`, { prefix: "Transfer" });
+              }
               return;
             } catch (error) {
               // File doesn't exist, proceed with transfer
@@ -998,7 +1067,7 @@ export class ComprehensiveTransfer {
   }
 
   /**
-   * Helper method to transfer documents between databases
+   * Helper method to transfer documents between databases using bulk operations with content and permission-based filtering
    */
   private async transferDocumentsBetweenDatabases(
     sourceDb: Databases,
@@ -1008,69 +1077,407 @@ export class ComprehensiveTransfer {
     sourceCollectionId: string,
     targetCollectionId: string
   ): Promise<void> {
-    MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+    MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId} with bulk operations, content comparison, and permission filtering`, { prefix: "Transfer" });
 
     let lastId: string | undefined;
     let totalTransferred = 0;
+    let totalSkipped = 0;
+    let totalUpdated = 0;
+
+    // Check if bulk operations are supported
+    const supportsBulk = this.options.sourceEndpoint.includes('cloud.appwrite.io') || 
+                        this.options.targetEndpoint.includes('cloud.appwrite.io');
+    
+    if (supportsBulk) {
+      MessageFormatter.info(`Using bulk operations for enhanced performance`, { prefix: "Transfer" });
+    }
 
     while (true) {
-      const queries = [Query.limit(50)]; // Smaller batch size for better performance
+      // Fetch source documents in larger batches (1000 instead of 50)
+      const queries = [Query.limit(1000)];
       if (lastId) {
         queries.push(Query.cursorAfter(lastId));
       }
 
-      const documents = await tryAwaitWithRetry(async () => 
+      const sourceDocuments = await tryAwaitWithRetry(async () => 
         sourceDb.listDocuments(sourceDbId, sourceCollectionId, queries)
       );
 
-      if (documents.documents.length === 0) {
+      if (sourceDocuments.documents.length === 0) {
         break;
       }
 
-      // Transfer documents with rate limiting
-      const transferTasks = documents.documents.map(doc => 
-        this.limit(async () => {
-          try {
-            // Check if document already exists
-            try {
-              await targetDb.getDocument(targetDbId, targetCollectionId, doc.$id);
-              MessageFormatter.info(`Document ${doc.$id} already exists, skipping`, { prefix: "Transfer" });
-              return;
-            } catch (error) {
-              // Document doesn't exist, proceed with creation
-            }
+      MessageFormatter.info(`Processing batch of ${sourceDocuments.documents.length} source documents`, { prefix: "Transfer" });
 
-            // Create document in target
-            const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
-            
-            await tryAwaitWithRetry(async () =>
-              targetDb.createDocument(
-                targetDbId,
-                targetCollectionId,
-                doc.$id,
-                docData,
-                doc.$permissions
-              )
-            );
+      // Extract document IDs from the current batch
+      const sourceDocIds = sourceDocuments.documents.map(doc => doc.$id);
+      
+      // Fetch existing documents from target in a single query
+      const existingTargetDocs = await this.fetchTargetDocumentsBatch(
+        targetDb, 
+        targetDbId, 
+        targetCollectionId, 
+        sourceDocIds
+      );
 
-            totalTransferred++;
-            MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
-          } catch (error) {
-            MessageFormatter.error(`Failed to transfer document ${doc.$id}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+      // Create a map for quick lookup of existing documents
+      const existingDocsMap = new Map<string, Models.Document>();
+      existingTargetDocs.forEach(doc => {
+        existingDocsMap.set(doc.$id, doc);
+      });
+
+      // Filter documents based on existence, content comparison, and permission comparison
+      const documentsToTransfer: Models.Document[] = [];
+      const documentsToUpdate: { doc: Models.Document; targetDoc: Models.Document; reason: string }[] = [];
+
+      for (const sourceDoc of sourceDocuments.documents) {
+        const existingTargetDoc = existingDocsMap.get(sourceDoc.$id);
+        
+        if (!existingTargetDoc) {
+          // Document doesn't exist in target, needs to be transferred
+          documentsToTransfer.push(sourceDoc);
+        } else {
+          // Document exists, compare both content and permissions
+          const sourcePermissions = JSON.stringify((sourceDoc.$permissions || []).sort());
+          const targetPermissions = JSON.stringify((existingTargetDoc.$permissions || []).sort());
+          const permissionsDiffer = sourcePermissions !== targetPermissions;
+          
+          // Use objectNeedsUpdate to compare document content (excluding system fields)
+          const contentDiffers = objectNeedsUpdate(existingTargetDoc, sourceDoc);
+          
+          if (contentDiffers && permissionsDiffer) {
+            // Both content and permissions differ
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "content and permissions differ"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but content and permissions differ - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else if (contentDiffers) {
+            // Only content differs
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "content differs"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but content differs - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else if (permissionsDiffer) {
+            // Only permissions differ
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "permissions differ"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but permissions differ - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else {
+            // Document exists with identical content AND permissions, skip
+            totalSkipped++;
+            MessageFormatter.info(`Document ${sourceDoc.$id} exists with matching content and permissions - skipping`, { prefix: "Transfer" });
           }
-        })
+        }
+      }
+
+      MessageFormatter.info(
+        `Batch analysis: ${documentsToTransfer.length} to create, ${documentsToUpdate.length} to update, ${totalSkipped} skipped so far`, 
+        { prefix: "Transfer" }
       );
 
-      await Promise.all(transferTasks);
+      // Process new documents with bulk operations if supported and available
+      if (documentsToTransfer.length > 0) {
+        if (supportsBulk && documentsToTransfer.length >= 10) {
+          // Use bulk operations for large batches
+          await this.transferDocumentsBulk(
+            targetDb,
+            targetDbId,
+            targetCollectionId,
+            documentsToTransfer
+          );
+          totalTransferred += documentsToTransfer.length;
+          MessageFormatter.success(`Bulk transferred ${documentsToTransfer.length} new documents`, { prefix: "Transfer" });
+        } else {
+          // Use individual transfers for smaller batches or non-bulk endpoints
+          const transferCount = await this.transferDocumentsIndividual(
+            targetDb,
+            targetDbId,
+            targetCollectionId,
+            documentsToTransfer
+          );
+          totalTransferred += transferCount;
+        }
+      }
 
-      if (documents.documents.length < 50) {
+      // Process document updates (always individual since bulk update with permissions needs special handling)
+      if (documentsToUpdate.length > 0) {
+        const updateCount = await this.updateDocumentsIndividual(
+          targetDb,
+          targetDbId,
+          targetCollectionId,
+          documentsToUpdate
+        );
+        totalUpdated += updateCount;
+      }
+
+      if (sourceDocuments.documents.length < 1000) {
         break;
       }
 
-      lastId = documents.documents[documents.documents.length - 1].$id;
+      lastId = sourceDocuments.documents[sourceDocuments.documents.length - 1].$id;
     }
 
-    MessageFormatter.info(`Transferred ${totalTransferred} documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+    MessageFormatter.info(
+      `Transfer complete: ${totalTransferred} new, ${totalUpdated} updated, ${totalSkipped} skipped from ${sourceCollectionId} to ${targetCollectionId}`, 
+      { prefix: "Transfer" }
+    );
+  }
+
+  /**
+   * Fetch target documents by IDs in batches to check existence and permissions
+   */
+  private async fetchTargetDocumentsBatch(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    docIds: string[]
+  ): Promise<Models.Document[]> {
+    const documents: Models.Document[] = [];
+    
+    // Split IDs into chunks of 100 for Query.equal limitations
+    const idChunks = this.chunkArray(docIds, 100);
+    
+    for (const chunk of idChunks) {
+      try {
+        const result = await tryAwaitWithRetry(async () => 
+          targetDb.listDocuments(targetDbId, targetCollectionId, [
+            Query.equal('$id', chunk),
+            Query.limit(100)
+          ])
+        );
+        documents.push(...result.documents);
+      } catch (error) {
+        // If query fails, fall back to individual gets (less efficient but more reliable)
+        MessageFormatter.warning(
+          `Batch query failed for ${chunk.length} documents, falling back to individual checks`, 
+          { prefix: "Transfer" }
+        );
+        
+        for (const docId of chunk) {
+          try {
+            const doc = await targetDb.getDocument(targetDbId, targetCollectionId, docId);
+            documents.push(doc);
+          } catch (getError) {
+            // Document doesn't exist, which is fine
+          }
+        }
+      }
+    }
+    
+    return documents;
+  }
+
+  /**
+   * Transfer documents using bulk operations with proper batch size handling
+   */
+  private async transferDocumentsBulk(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documents: Models.Document[]
+  ): Promise<void> {
+    // Prepare documents for bulk upsert
+    const preparedDocs = documents.map(doc => {
+      const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+      return {
+        $id,
+        $permissions,
+        ...docData
+      };
+    });
+
+    // Process in smaller chunks for bulk operations (1000 for Pro, 100 for Free tier)
+    const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
+    let processed = false;
+
+    for (const maxBatchSize of batchSizes) {
+      const documentBatches = this.chunkArray(preparedDocs, maxBatchSize);
+      
+      try {
+        for (const batch of documentBatches) {
+          MessageFormatter.info(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
+          
+          await this.bulkUpsertDocuments(
+            this.targetClient,
+            targetDbId,
+            targetCollectionId,
+            batch
+          );
+          
+          MessageFormatter.success(`✅ Bulk upserted ${batch.length} documents`, { prefix: "Transfer" });
+          
+          // Add delay between batches to respect rate limits
+          if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+            await new Promise(resolve => setTimeout(resolve, 200));
+          }
+        }
+        
+        processed = true;
+        break; // Success, exit batch size loop
+      } catch (error) {
+        MessageFormatter.warning(
+          `Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`, 
+          { prefix: "Transfer" }
+        );
+        continue; // Try next smaller batch size
+      }
+    }
+
+    if (!processed) {
+      MessageFormatter.warning(
+        `All bulk operations failed, falling back to individual transfers`, 
+        { prefix: "Transfer" }
+      );
+      
+      // Fall back to individual transfers
+      await this.transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documents);
+    }
+  }
+
+  /**
+   * Direct HTTP implementation of bulk upsert API
+   */
+  private async bulkUpsertDocuments(
+    client: any,
+    dbId: string,
+    collectionId: string,
+    documents: any[]
+  ): Promise<any> {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    
+    const headers = {
+      'Content-Type': 'application/json',
+      'X-Appwrite-Project': client.config.project,
+      'X-Appwrite-Key': client.config.key
+    };
+    
+    const response = await fetch(url.toString(), {
+      method: 'PUT',
+      headers,
+      body: JSON.stringify({ documents })
+    });
+    
+    if (!response.ok) {
+      const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+      throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    
+    return await response.json();
+  }
+
+  /**
+   * Transfer documents individually with rate limiting
+   */
+  private async transferDocumentsIndividual(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documents: Models.Document[]
+  ): Promise<number> {
+    let successCount = 0;
+
+    const transferTasks = documents.map(doc => 
+      this.limit(async () => {
+        try {
+          const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+          
+          await tryAwaitWithRetry(async () =>
+            targetDb.createDocument(
+              targetDbId,
+              targetCollectionId,
+              doc.$id,
+              docData,
+              doc.$permissions
+            )
+          );
+
+          successCount++;
+          MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
+        } catch (error) {
+          MessageFormatter.error(
+            `Failed to transfer document ${doc.$id}`, 
+            error instanceof Error ? error : new Error(String(error)), 
+            { prefix: "Transfer" }
+          );
+        }
+      })
+    );
+
+    await Promise.all(transferTasks);
+    return successCount;
+  }
+
+  /**
+   * Update documents individually with content and/or permission changes
+   */
+  private async updateDocumentsIndividual(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documentPairs: { doc: Models.Document; targetDoc: Models.Document; reason: string }[]
+  ): Promise<number> {
+    let successCount = 0;
+
+    const updateTasks = documentPairs.map(({ doc, targetDoc, reason }) => 
+      this.limit(async () => {
+        try {
+          const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+          
+          await tryAwaitWithRetry(async () =>
+            targetDb.updateDocument(
+              targetDbId,
+              targetCollectionId,
+              doc.$id,
+              docData,
+              doc.$permissions
+            )
+          );
+
+          successCount++;
+          MessageFormatter.success(
+            `Updated document ${doc.$id} (${reason}) - permissions: [${targetDoc.$permissions?.join(', ')}] → [${doc.$permissions?.join(', ')}]`, 
+            { prefix: "Transfer" }
+          );
+        } catch (error) {
+          MessageFormatter.error(
+            `Failed to update document ${doc.$id} (${reason})`, 
+            error instanceof Error ? error : new Error(String(error)), 
+            { prefix: "Transfer" }
+          );
+        }
+      })
+    );
+
+    await Promise.all(updateTasks);
+    return successCount;
+  }
+
+  /**
+   * Utility method to chunk arrays
+   */
+  private chunkArray<T>(array: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
   }
 
   /**
@@ -1160,10 +1567,39 @@ export class ComprehensiveTransfer {
       const transferTasks = memberships.map(membership => 
         this.userLimit(async () => { // Use userLimit for team operations (more sensitive)
           try {
-            // Check if membership already exists
+            // Check if membership already exists and compare roles
+            let existingMembership: Models.Membership | null = null;
             try {
-              await this.targetTeams.getMembership(teamId, membership.$id);
-              MessageFormatter.info(`Membership ${membership.$id} already exists, skipping`, { prefix: "Transfer" });
+              existingMembership = await this.targetTeams.getMembership(teamId, membership.$id);
+              
+              // Compare roles between source and target membership
+              const sourceRoles = JSON.stringify(membership.roles?.sort() || []);
+              const targetRoles = JSON.stringify(existingMembership.roles?.sort() || []);
+              
+              if (sourceRoles !== targetRoles) {
+                MessageFormatter.warning(
+                  `Membership ${membership.$id} exists but has different roles. Source: ${sourceRoles}, Target: ${targetRoles}`, 
+                  { prefix: "Transfer" }
+                );
+                
+                // Update membership roles to match source
+                try {
+                  await this.targetTeams.updateMembership(
+                    teamId,
+                    membership.$id,
+                    membership.roles
+                  );
+                  MessageFormatter.success(`Updated membership ${membership.$id} roles to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update roles for membership ${membership.$id}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`Membership ${membership.$id} already exists with matching roles, skipping`, { prefix: "Transfer" });
+              }
               return;
             } catch (error) {
               // Membership doesn't exist, proceed with creation