appwrite-utils-cli 1.2.0 → 1.2.1

package/dist/collections/methods.js

@@ -413,9 +413,53 @@ const supportsBulkOperations = (endpoint) => {
     return endpoint.includes('cloud.appwrite.io');
 };
 /**
- * Enhanced bulk document creation using the new bulk upsert API for cloud.appwrite.io
+ * Direct HTTP implementation of bulk upsert API
  */
-const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, maxBatchSize = 1000) => {
+const bulkUpsertDocuments = async (client, dbId, collectionId, documents) => {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    const headers = {
+        'Content-Type': 'application/json',
+        'X-Appwrite-Project': client.config.project,
+        'X-Appwrite-Key': client.config.key
+    };
+    const response = await fetch(url.toString(), {
+        method: 'PUT',
+        headers,
+        body: JSON.stringify({ documents })
+    });
+    if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
+        throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    return await response.json();
+};
+/**
+ * Direct HTTP implementation of bulk create API
+ */
+const bulkCreateDocuments = async (client, dbId, collectionId, documents) => {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    const headers = {
+        'Content-Type': 'application/json',
+        'X-Appwrite-Project': client.config.project,
+        'X-Appwrite-Key': client.config.key
+    };
+    const response = await fetch(url.toString(), {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({ documents })
+    });
+    if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
+        throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    return await response.json();
+};
+/**
+ * Enhanced bulk document creation using direct HTTP calls
+ */
+const transferDocumentsBulkUpsert = async (client, dbId, collectionId, documents, maxBatchSize = 1000) => {
     let successful = 0;
     let failed = 0;
     // Prepare documents for bulk upsert
@@ -426,7 +470,6 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
         delete toCreateObject.$createdAt;
         delete toCreateObject.$updatedAt;
         // Keep $id and $permissions for upsert functionality
-        // Appwrite bulk API expects $permissions to be preserved
         return toCreateObject;
     });
     // Process in batches based on plan limits
@@ -434,8 +477,8 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
     for (const batch of documentBatches) {
         console.log(chalk.blue(`Bulk upserting ${batch.length} documents...`));
         try {
-            // Try bulk upsert with the highest batch size first
-            const result = await db.upsertDocuments(dbId, collectionId, batch);
+            // Try bulk upsert with direct HTTP call
+            const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
             successful += result.documents?.length || batch.length;
             console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || batch.length} documents`));
         }
@@ -446,13 +489,14 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
                 const smallerBatches = chunk(batch, 100);
                 for (const smallBatch of smallerBatches) {
                     try {
-                        const result = await db.upsertDocuments(dbId, collectionId, smallBatch);
+                        const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
                         successful += result.documents?.length || smallBatch.length;
                         console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`));
                     }
                     catch (smallBatchError) {
                         console.log(chalk.yellow(`Smaller batch failed, falling back to individual transfers...`));
                         // Fall back to individual document transfer for this batch
+                        const db = new Databases(client);
                         const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, smallBatch.map((doc, index) => ({
                             ...doc,
                             $id: documents[documentBatches.indexOf(batch) * maxBatchSize + smallerBatches.indexOf(smallBatch) * 100 + index]?.$id || ID.unique(),
@@ -467,6 +511,7 @@ const transferDocumentsBulkUpsert = async (db, dbId, collectionId, documents, ma
             }
             else {
                 // Fall back to individual document transfer
+                const db = new Databases(client);
                 const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(db, dbId, collectionId, batch.map((doc, index) => ({
                     ...doc,
                     $id: documents[documentBatches.indexOf(batch) * maxBatchSize + index]?.$id || ID.unique(),
@@ -529,15 +574,15 @@ const transferDocumentBatchWithRetryFallback = async (db, dbId, collectionId, do
 /**
  * Enhanced batch document transfer with fault tolerance and bulk API support
  */
-const transferDocumentBatchWithRetry = async (db, endpoint, dbId, collectionId, documents, batchSize = 10) => {
+const transferDocumentBatchWithRetry = async (db, client, dbId, collectionId, documents, batchSize = 10) => {
     // Check if we can use bulk operations
-    if (supportsBulkOperations(endpoint)) {
+    if (supportsBulkOperations(client.config.endpoint)) {
         console.log(chalk.green(`🚀 Using bulk upsert API for faster document transfer`));
         // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
         const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
         for (const maxBatchSize of batchSizes) {
             try {
-                return await transferDocumentsBulkUpsert(db, dbId, collectionId, documents, maxBatchSize);
+                return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
             }
             catch (error) {
                 console.log(chalk.yellow(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`));
@@ -574,7 +619,7 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint
             break;
         }
         console.log(chalk.blue(`Fetched ${fromCollDocs.documents.length} documents, processing for transfer...`));
-        const { successful, failed } = await transferDocumentBatchWithRetry(remoteDb, endpoint, toDbId, toCollId, fromCollDocs.documents);
+        const { successful, failed } = await transferDocumentBatchWithRetry(remoteDb, client, toDbId, toCollId, fromCollDocs.documents);
         totalDocumentsProcessed += fromCollDocs.documents.length;
         totalSuccessful += successful;
         totalFailed += failed;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "main": "src/main.ts",
   "type": "module",
   "repository": {
@@ -43,7 +43,7 @@
     "js-yaml": "^4.1.0",
     "luxon": "^3.6.1",
     "nanostores": "^0.10.3",
-    "node-appwrite": "^17",
+    "node-appwrite": "^16",
     "p-limit": "^6.2.0",
     "tar": "^7.4.3",
     "tsx": "^4.20.3",

package/src/collections/methods.ts

@@ -677,10 +677,74 @@ const supportsBulkOperations = (endpoint: string): boolean => {
 };
 
 /**
- * Enhanced bulk document creation using the new bulk upsert API for cloud.appwrite.io
+ * Direct HTTP implementation of bulk upsert API
+ */
+const bulkUpsertDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+  
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+  
+  const response = await fetch(url.toString(), {
+    method: 'PUT',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+  
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+  
+  return await response.json();
+};
+
+/**
+ * Direct HTTP implementation of bulk create API
+ */
+const bulkCreateDocuments = async (
+  client: any,
+  dbId: string,
+  collectionId: string,
+  documents: any[]
+): Promise<any> => {
+  const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+  const url = new URL(client.config.endpoint + apiPath);
+  
+  const headers = {
+    'Content-Type': 'application/json',
+    'X-Appwrite-Project': client.config.project,
+    'X-Appwrite-Key': client.config.key
+  };
+  
+  const response = await fetch(url.toString(), {
+    method: 'POST',
+    headers,
+    body: JSON.stringify({ documents })
+  });
+  
+  if (!response.ok) {
+    const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+    throw new Error(`Bulk create failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+  }
+  
+  return await response.json();
+};
+
+/**
+ * Enhanced bulk document creation using direct HTTP calls
  */
 const transferDocumentsBulkUpsert = async (
-  db: Databases,
+  client: any,
   dbId: string,
   collectionId: string,
   documents: any[],
@@ -698,7 +762,6 @@ const transferDocumentsBulkUpsert = async (
     delete toCreateObject.$updatedAt;
     
     // Keep $id and $permissions for upsert functionality
-    // Appwrite bulk API expects $permissions to be preserved
     return toCreateObject;
   });
   
@@ -709,8 +772,8 @@ const transferDocumentsBulkUpsert = async (
     console.log(chalk.blue(`Bulk upserting ${batch.length} documents...`));
     
     try {
-      // Try bulk upsert with the highest batch size first
-      const result = await (db as any).upsertDocuments(dbId, collectionId, batch);
+      // Try bulk upsert with direct HTTP call
+      const result = await bulkUpsertDocuments(client, dbId, collectionId, batch);
       successful += result.documents?.length || batch.length;
       console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || batch.length} documents`));
       
@@ -723,13 +786,14 @@ const transferDocumentsBulkUpsert = async (
         
         for (const smallBatch of smallerBatches) {
           try {
-            const result = await (db as any).upsertDocuments(dbId, collectionId, smallBatch);
+            const result = await bulkUpsertDocuments(client, dbId, collectionId, smallBatch);
             successful += result.documents?.length || smallBatch.length;
             console.log(chalk.green(`✅ Bulk upserted ${result.documents?.length || smallBatch.length} documents (smaller batch)`));
           } catch (smallBatchError: any) {
             console.log(chalk.yellow(`Smaller batch failed, falling back to individual transfers...`));
             
             // Fall back to individual document transfer for this batch
+            const db = new Databases(client);
             const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
               db, dbId, collectionId, smallBatch.map((doc, index) => ({
                 ...doc,
@@ -746,6 +810,7 @@ const transferDocumentsBulkUpsert = async (
         }
       } else {
         // Fall back to individual document transfer
+        const db = new Databases(client);
         const { successful: indivSuccessful, failed: indivFailed } = await transferDocumentBatchWithRetryFallback(
           db, dbId, collectionId, batch.map((doc, index) => ({
             ...doc,
@@ -836,14 +901,14 @@ const transferDocumentBatchWithRetryFallback = async (
  */
 const transferDocumentBatchWithRetry = async (
   db: Databases,
-  endpoint: string,
+  client: any,
   dbId: string,
   collectionId: string,
   documents: any[],
   batchSize: number = 10
 ): Promise<{ successful: number; failed: number }> => {
   // Check if we can use bulk operations
-  if (supportsBulkOperations(endpoint)) {
+  if (supportsBulkOperations(client.config.endpoint)) {
     console.log(chalk.green(`🚀 Using bulk upsert API for faster document transfer`));
     
     // Try with Scale plan limit first (2500), then Pro (1000), then Free (100)
@@ -851,7 +916,7 @@ const transferDocumentBatchWithRetry = async (
     
     for (const maxBatchSize of batchSizes) {
       try {
-        return await transferDocumentsBulkUpsert(db, dbId, collectionId, documents, maxBatchSize);
+        return await transferDocumentsBulkUpsert(client, dbId, collectionId, documents, maxBatchSize);
       } catch (error: any) {
         console.log(chalk.yellow(`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`));
         continue;
@@ -911,7 +976,7 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
     
     const { successful, failed } = await transferDocumentBatchWithRetry(
       remoteDb,
-      endpoint,
+      client,
       toDbId,
       toCollId,
       fromCollDocs.documents