appwrite-utils-cli 1.2.0 → 1.2.2

package/dist/collections/methods.js

@@ -413,9 +413,53 @@ const supportsBulkOperations = (endpoint) => {
     return endpoint.includes('cloud.appwrite.io');
 };
 /**
- * Enhanced bulk document creation using the new bulk upsert API for cloud.appwrite.io
+ * Direct HTTP implementation of bulk upsert API
  */
-const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, maxBatchSize = 1000) => {
+const bulkUpsertDocuments = async (client, dbId, collectionId, documents) => {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    const headers = {
+        'Content-Type': 'application/json',
+        'X-Appwrite-Project': client.config.project,
+        'X-Appwrite-Key': client.config.key
+    };
+    const response = await fetch(url.toString(), {
+        method: 'PUT',
+        headers,
+        body: JSON.stringify({ documents })
+    });
+    if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
+        throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    return await response.json();
+};
+/**
+ * Direct HTTP implementation of bulk create API
+ */
+const bulkCreateDocuments = async (client, dbId, collectionId, documents) => {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    const headers = {
+        'Content-Type': 'application/json',
+        'X-Appwrite-Project': client.config.project,
+        'X-Appwrite-Key': client.config.key
+    };
+    const response = await fetch(url.toString(), {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({ documents })
+    });
+    if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
+        throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    return await response.json();
+};
+/**
+ * Enhanced bulk document creation using direct HTTP calls
+ */
+const transferDocumentsBulkUpsert = async (client, dbId, collectionId, documents, maxBatchSize = 1000) => {
     let successful = 0;
     let failed = 0;
     // Prepare documents for bulk upsert
@@ -426,7 +470,6 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
         delete toCreateObject.$createdAt;
         delete toCreateObject.$updatedAt;
         // Keep $id and $permissions for upsert functionality
-        // Appwrite bulk API expects $permissions to be preserved
         return toCreateObject;
     });
     // Process in batches based on plan limits
@@ -434,8 +477,8 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
     for (const batch of documentBatches) {
         console.log(chalk.blue(`Bulk upserting ${batch.length} documents...`));
         try {
-            // Try bulk upsert with the highest batch size first
-            const result = await db.upsertDocuments(dbId, collectionId, batch);
+            // Try bulk upsert with direct HTTP call
+            const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
             successful += result.documents?.length || batch.length;
             console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || batch.length} documents`));
         }
@@ -446,13 +489,14 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
                 const smallerBatches = chunk(batch, 100);
                 for (const smallBatch of smallerBatches) {
                     try {
-                        const result = await db.upsertDocuments(dbId, collectionId, smallBatch);
+                        const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
                         successful += result.documents?.length || smallBatch.length;
                         console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`));
                     }
                     catch (smallBatchError) {
                         console.log(chalk.yellow(`Smaller batch failed, falling back to individual transfers...`));
                         // Fall back to individual document transfer for this batch
+                        const db = new Databases(client);
                         const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, smallBatch.map((doc, index) => ({
                             ...doc,
                             $id: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$id || ID.unique(),
@@ -467,6 +511,7 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
             }
             else {
                 // Fall back to individual document transfer
+                const db = new Databases(client);
                 const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, batch.map((doc, index) => ({
                     ...doc,
                     $id: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$id || ID.unique(),
@@ -529,15 +574,15 @@ const transferDocumentBatchWithRetryFallback = async (db, dbId, collectionId, do
 /**
  * Enhanced batch document transfer with fault tolerance and bulk API support
  */
-const transferDocumentBatchWithRetry = async (db, endpoint, dbId, collectionId, documents, batchSize = 10) => {
+const transferDocumentBatchWithRetry = async (db, client, dbId, collectionId, documents, batchSize = 10) => {
     // Check if we can use bulk operations
-    if (supportsBulkOperations(endpoint)) {
+    if (supportsBulkOperations(client.config.endpoint)) {
         console.log(chalk.green(`🚀 Using bulk upsert API for faster document transfer`));
         // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
         const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
         for (const maxBatchSize of batchSizes) {
             try {
-                return await transferDocumentsBulkUpsert(db, dbId, collectionId, documents, maxBatchSize);
+                return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
             }
             catch (error) {
                 console.log(chalk.yellow(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`));
@@ -574,7 +619,7 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint
             break;
         }
         console.log(chalk.blue(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`));
-        const { successful, failed } = await transferDocumentBatchWithRetry(remoteDb, endpoint, toDbId, toCollId, fromCollDocs.documents);
+        const { successful, failed } = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
         totalDocumentsProcessed += fromCollDocs.documents.length;
         totalSuccessful += successful;
         totalFailed += failed;

package/dist/migrations/comprehensiveTransfer.d.ts

@@ -74,6 +74,10 @@ export declare class ComprehensiveTransfer {
      * Helper method to fetch all collections from a database
      */
     private fetchAllCollections;
+    /**
+     * Helper method to fetch all buckets with pagination
+     */
+    private fetchAllBuckets;
     /**
      * Helper method to parse attribute objects (simplified version of parseAttribute)
      */

package/dist/migrations/comprehensiveTransfer.js

@@ -272,21 +272,22 @@ export class ComprehensiveTransfer {
     async transferAllBuckets() {
         MessageFormatter.info("Starting bucket transfer phase", { prefix: "Transfer" });
         try {
-            const sourceBuckets = await this.sourceStorage.listBuckets();
-            const targetBuckets = await this.targetStorage.listBuckets();
+            // Get all buckets from source with pagination
+            const allSourceBuckets = await this.fetchAllBuckets(this.sourceStorage);
+            const allTargetBuckets = await this.fetchAllBuckets(this.targetStorage);
             if (this.options.dryRun) {
                 let totalFiles = 0;
-                for (const bucket of sourceBuckets.buckets) {
+                for (const bucket of allSourceBuckets) {
                     const files = await this.sourceStorage.listFiles(bucket.$id, [Query.limit(1)]);
                     totalFiles += files.total;
                 }
-                MessageFormatter.info(`DRY RUN: Would transfer ${sourceBuckets.buckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
+                MessageFormatter.info(`DRY RUN: Would transfer ${allSourceBuckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
                 return;
             }
-            const transferTasks = sourceBuckets.buckets.map(bucket => this.limit(async () => {
+            const transferTasks = allSourceBuckets.map(bucket => this.limit(async () => {
                 try {
                     // Check if bucket exists in target
-                    const existingBucket = targetBuckets.buckets.find(tb => tb.$id === bucket.$id);
+                    const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
                     if (!existingBucket) {
                         // Create bucket in target
                         await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
@@ -480,6 +481,29 @@ export class ComprehensiveTransfer {
         }
         return collections;
     }
+    /**
+     * Helper method to fetch all buckets with pagination
+     */
+    async fetchAllBuckets(storage) {
+        const buckets = [];
+        let lastId;
+        while (true) {
+            const queries = [Query.limit(100)];
+            if (lastId) {
+                queries.push(Query.cursorAfter(lastId));
+            }
+            const result = await tryAwaitWithRetry(async () => storage.listBuckets(queries));
+            if (result.buckets.length === 0) {
+                break;
+            }
+            buckets.push(...result.buckets);
+            if (result.buckets.length < 100) {
+                break;
+            }
+            lastId = result.buckets[result.buckets.length - 1].$id;
+        }
+        return buckets;
+    }
     /**
      * Helper method to parse attribute objects (simplified version of parseAttribute)
      */

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "main": "src/main.ts",
   "type": "module",
   "repository": {
@@ -43,7 +43,7 @@
     "js-yaml": "^4.1.0",
     "luxon": "^3.6.1",
     "nanostores": "^0.10.3",
-    "node-appwrite": "^17",
+    "node-appwrite": "^16",
     "p-limit": "^6.2.0",
     "tar": "^7.4.3",
     "tsx": "^4.20.3",

package/src/collections/methods.ts

@@ -677,10 +677,74 @@ const supportsBulkOperations = (endpoint: string): boolean => {
 };
 
 /**
- * Enhanced bulk document creation using the new bulk upsert API for cloud.appwrite.io
+ * Direct HTTP implementation of bulk upsert API
+ */
+const bulkUpsertDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+  
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+  
+  const response = await fetch(url.toString(), {
+    method: 'PUT',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+  
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+  
+  return await response.json();
+};
+
+/**
+ * Direct HTTP implementation of bulk create API
+ */
+const bulkCreateDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+  
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+  
+  const response = await fetch(url.toString(), {
+    method: 'POST',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+  
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+  
+  return await response.json();
+};
+
+/**
+ * Enhanced bulk document creation using direct HTTP calls
  */
 const transferDocumentsBulkUpsert = async (
-  db: Databases,
+  client: any,
   dbId: string,
   collectionId: string,
   documents: any[],
@@ -698,7 +762,6 @@ const transferDocumentsBulkUpsert = async (
     delete toCreateObject.$updatedAt;
     
     // Keep $id and $permissions for upsert functionality
-    // Appwrite bulk API expects $permissions to be preserved
     return toCreateObject;
   });
   
@@ -709,8 +772,8 @@ const transferDocumentsBulkUpsert = async (
     console.log(chalk.blue(`Bulk upserting ${batch.length} documents...`));
     
     try {
-      // Try bulk upsert with the highest batch size first
-      const result = await (db as any).upsertDocuments(dbId, collectionId, batch);
+      // Try bulk upsert with direct HTTP call
+      const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
       successful += result.documents?.length || batch.length;
       console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || batch.length} documents`));
       
@@ -723,13 +786,14 @@ const transferDocumentsBulkUpsert = async (
         
         for (const smallBatch of smallerBatches) {
           try {
-            const result = await (db as any).upsertDocuments(dbId, collectionId, smallBatch);
+            const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
             successful += result.documents?.length || smallBatch.length;
             console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`));
           } catch (smallBatchError: any) {
             console.log(chalk.yellow(`Smaller batch failed, falling back to individual transfers...`));
             
             // Fall back to individual document transfer for this batch
+            const db = new Databases(client);
             const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
               db, dbId, collectionId, smallBatch.map((doc, index) => ({
                 ...doc,
@@ -746,6 +810,7 @@ const transferDocumentsBulkUpsert = async (
         }
       } else {
         // Fall back to individual document transfer
+        const db = new Databases(client);
         const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
           db, dbId, collectionId, batch.map((doc, index) => ({
             ...doc,
@@ -836,14 +901,14 @@ const transferDocumentBatchWithRetryFallback = async (
  */
 const transferDocumentBatchWithRetry = async (
   db: Databases,
-  endpoint: string,
+  client: any,
   dbId: string,
   collectionId: string,
   documents: any[],
   batchSize: number = 10
 ): Promise<{ successful: number; failed: number }> => {
   // Check if we can use bulk operations
-  if (supportsBulkOperations(endpoint)) {
+  if (supportsBulkOperations(client.config.endpoint)) {
     console.log(chalk.green(`🚀 Using bulk upsert API for faster document transfer`));
     
     // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
@@ -851,7 +916,7 @@ const transferDocumentBatchWithRetry = async (
     
     for (const maxBatchSize of batchSizes) {
       try {
-        return await transferDocumentsBulkUpsert(db, dbId, collectionId, documents, maxBatchSize);
+        return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
       } catch (error: any) {
         console.log(chalk.yellow(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`));
         continue;
@@ -911,7 +976,7 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
     
     const { successful, failed } = await transferDocumentBatchWithRetry(
       remoteDb,
-      endpoint,
+      client,
       toDbId,
       toCollId,
       fromCollDocs.documents

package/src/migrations/comprehensiveTransfer.ts

@@ -408,24 +408,25 @@ export class ComprehensiveTransfer {
     MessageFormatter.info("Starting bucket transfer phase", { prefix: "Transfer" });
 
     try {
-      const sourceBuckets = await this.sourceStorage.listBuckets();
-      const targetBuckets = await this.targetStorage.listBuckets();
+      // Get all buckets from source with pagination
+      const allSourceBuckets = await this.fetchAllBuckets(this.sourceStorage);
+      const allTargetBuckets = await this.fetchAllBuckets(this.targetStorage);
 
       if (this.options.dryRun) {
         let totalFiles = 0;
-        for (const bucket of sourceBuckets.buckets) {
+        for (const bucket of allSourceBuckets) {
           const files = await this.sourceStorage.listFiles(bucket.$id, [Query.limit(1)]);
           totalFiles += files.total;
         }
-        MessageFormatter.info(`DRY RUN: Would transfer ${sourceBuckets.buckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
+        MessageFormatter.info(`DRY RUN: Would transfer ${allSourceBuckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
         return;
       }
 
-      const transferTasks = sourceBuckets.buckets.map(bucket => 
+      const transferTasks = allSourceBuckets.map(bucket => 
         this.limit(async () => {
           try {
             // Check if bucket exists in target
-            const existingBucket = targetBuckets.buckets.find(tb => tb.$id === bucket.$id);
+            const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
             
             if (!existingBucket) {
               // Create bucket in target
@@ -676,6 +677,37 @@ export class ComprehensiveTransfer {
     return collections;
   }
 
+  /**
+   * Helper method to fetch all buckets with pagination
+   */
+  private async fetchAllBuckets(storage: Storage): Promise<Models.Bucket[]> {
+    const buckets: Models.Bucket[] = [];
+    let lastId: string | undefined;
+
+    while (true) {
+      const queries = [Query.limit(100)];
+      if (lastId) {
+        queries.push(Query.cursorAfter(lastId));
+      }
+
+      const result = await tryAwaitWithRetry(async () => storage.listBuckets(queries));
+      
+      if (result.buckets.length === 0) {
+        break;
+      }
+
+      buckets.push(...result.buckets);
+      
+      if (result.buckets.length < 100) {
+        break;
+      }
+      
+      lastId = result.buckets[result.buckets.length - 1].$id;
+    }
+
+    return buckets;
+  }
+
   /**
    * Helper method to parse attribute objects (simplified version of parseAttribute)
    */