appwrite-utils-cli 0.0.37 → 0.0.39

package/src/migrations/migrationHelper.ts

@@ -3,6 +3,7 @@ import { BatchSchema, OperationSchema, type Operation } from "./backup.js";
 import { AttributeMappingsSchema } from "appwrite-utils";
 import { z } from "zod";
 import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "src/utils/helperFunctions.js";
 
 /**
  * Object that contains the context for an action that needs to be executed after import
@@ -148,11 +149,14 @@ export const updateOperation = async (
   operationId: string,
   updateFields: any
 ) => {
-  await database.updateDocument(
-    "migrations",
-    "currentOperations",
-    operationId,
-    updateFields
+  await tryAwaitWithRetry(
+    async () =>
+      await database.updateDocument(
+        "migrations",
+        "currentOperations",
+        operationId,
+        updateFields
+      )
   );
 };
 

package/src/migrations/queue.ts

@@ -3,6 +3,7 @@ import type { Attribute } from "appwrite-utils";
 import { createOrUpdateAttribute } from "./attributes.js";
 import _ from "lodash";
 import { fetchAndCacheCollectionByName } from "./collections.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export interface QueuedOperation {
   type: "attribute";
@@ -37,9 +38,8 @@ export const processQueue = async (db: Databases, dbId: string) => {
         if (operation.collectionId) {
           console.log(`\tFetching collection by ID: ${operation.collectionId}`);
           try {
-            collectionFound = await db.getCollection(
-              dbId,
-              operation.collectionId
+            collectionFound = await tryAwaitWithRetry(
+              async () => await db.getCollection(dbId, operation.collectionId!)
             );
           } catch (e) {
             console.log(
@@ -70,9 +70,8 @@ export const processQueue = async (db: Databases, dbId: string) => {
         // Handle non-relationship operations with a specified collectionId
         console.log(`\tFetching collection by ID: ${operation.collectionId}`);
         try {
-          collectionFound = await db.getCollection(
-            dbId,
-            operation.collectionId
+          collectionFound = await tryAwaitWithRetry(
+            async () => await db.getCollection(dbId, operation.collectionId!)
           );
         } catch (e) {
           console.log(

package/src/migrations/setupDatabase.ts

@@ -6,7 +6,11 @@ import {
   wipeDatabase,
 } from "./collections.js";
 import { getMigrationCollectionSchemas } from "./backup.js";
-import { areCollectionNamesSame, toCamelCase } from "../utils/index.js";
+import {
+  areCollectionNamesSame,
+  toCamelCase,
+  tryAwaitWithRetry,
+} from "../utils/index.js";
 import {
   backupDatabase,
   initOrGetBackupStorage,
@@ -28,16 +32,18 @@ export const setupMigrationDatabase = async (config: AppwriteConfig) => {
   const dbCollections: Models.Collection[] = [];
   const migrationCollectionsSetup = getMigrationCollectionSchemas();
   try {
-    db = await database.get("migrations");
+    db = await tryAwaitWithRetry(async () => await database.get("migrations"));
     console.log("Migrations database found");
   } catch (e) {
-    db = await database.create("migrations", "Migrations", true);
+    db = await tryAwaitWithRetry(
+      async () => await database.create("migrations", "Migrations", true)
+    );
     console.log("Migrations database created");
   }
   if (db) {
-    const collectionsPulled = await database.listCollections(db.$id, [
-      Query.limit(25),
-    ]);
+    const collectionsPulled = await tryAwaitWithRetry(
+      async () => await database.listCollections(db.$id, [Query.limit(25)])
+    );
     dbCollections.push(...collectionsPulled.collections);
   }
   console.log(`Collections in migrations database: ${dbCollections.length}`);
@@ -49,19 +55,24 @@ export const setupMigrationDatabase = async (config: AppwriteConfig) => {
     const collectionId = toCamelCase(collectionName); // Convert name to toCamelCase for the ID
     let collectionFound: Models.Collection | null = null;
     try {
-      collectionFound = await database.getCollection(db.$id, collectionId);
+      collectionFound = await tryAwaitWithRetry(
+        async () => await database.getCollection(db.$id, collectionId)
+      );
     } catch (e) {
       console.log(`Collection not found: ${collectionId}`);
     }
     if (!collectionFound) {
       // Create the collection with the provided configuration
-      collectionFound = await database.createCollection(
-        db.$id,
-        collectionId,
-        collectionName,
-        undefined,
-        collection.documentSecurity,
-        collection.enabled
+      collectionFound = await tryAwaitWithRetry(
+        async () =>
+          await database.createCollection(
+            db.$id,
+            collectionId,
+            collectionName,
+            undefined,
+            collection.documentSecurity,
+            collection.enabled
+          )
       );
     }
     for (const attribute of attributes) {
@@ -87,11 +98,15 @@ export const ensureDatabasesExist = async (config: AppwriteConfig) => {
   });
   const dbNames = databasesToEnsure.map((db) => db.name);
 
-  const existingDatabases = await database.list([Query.equal("name", dbNames)]);
+  const existingDatabases = await tryAwaitWithRetry(
+    async () => await database.list([Query.equal("name", dbNames)])
+  );
 
   for (const db of databasesToEnsure) {
     if (!existingDatabases.databases.some((d) => d.name === db.name)) {
-      await database.create(db.$id || ID.unique(), db.name, true);
+      await tryAwaitWithRetry(
+        async () => await database.create(db.$id || ID.unique(), db.name, true)
+      );
       console.log(`${db.name} database created`);
     }
   }
@@ -106,12 +121,14 @@ export const wipeOtherDatabases = async (
   );
   databasesToKeep.push("migrations");
   console.log(`Databases to keep: ${databasesToKeep.join(", ")}`);
-  const allDatabases = await database.list([Query.limit(500)]);
+  const allDatabases = await tryAwaitWithRetry(
+    async () => await database.list([Query.limit(500)])
+  );
   for (const db of allDatabases.databases) {
     if (
       !databasesToKeep.includes(db.name.toLowerCase().trim().replace(" ", ""))
     ) {
-      await database.delete(db.$id);
+      await tryAwaitWithRetry(async () => await database.delete(db.$id));
       console.log(`Deleted database: ${db.name}`);
     }
   }

package/src/migrations/storage.ts

@@ -10,6 +10,7 @@ import {
 import { type OperationCreate, type BackupCreate } from "./backup.js";
 import { splitIntoBatches } from "./migrationHelper.js";
 import type { AppwriteConfig } from "appwrite-utils";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export const logOperation = async (
   db: Databases,
@@ -21,11 +22,14 @@ export const logOperation = async (
     let operation;
     if (operationId) {
       // Update existing operation log
-      operation = await db.updateDocument(
-        "migrations",
-        "currentOperations",
-        operationId,
-        operationDetails
+      operation = await tryAwaitWithRetry(
+        async () =>
+          await db.updateDocument(
+            "migrations",
+            "currentOperations",
+            operationId,
+            operationDetails
+          )
       );
     } else {
       // Create new operation log
@@ -46,14 +50,15 @@ export const logOperation = async (
 
 export const initOrGetBackupStorage = async (storage: Storage) => {
   try {
-    const backupStorage = await storage.getBucket("backupStorage");
+    const backupStorage = await tryAwaitWithRetry(
+      async () => await storage.getBucket("backupStorage")
+    );
     return backupStorage;
   } catch (e) {
     // ID backupStorage
     // Name Backups Storage
-    const backupStorage = await storage.createBucket(
-      "backupStorage",
-      "Backups Storage"
+    const backupStorage = await tryAwaitWithRetry(
+      async () => await storage.createBucket("backupStorage", "Backups Storage")
     );
     return backupStorage;
   }
@@ -65,21 +70,27 @@ export const initOrGetDocumentStorage = async (
   dbName: string
 ) => {
   try {
-    await storage.getBucket(
-      `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`
+    await tryAwaitWithRetry(
+      async () =>
+        await storage.getBucket(
+          `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`
+        )
     );
   } catch (e) {
     // ID documentStorage
     // Name Document Storage
-    const documentStorage = await storage.createBucket(
-      `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`,
-      `Document Storage ${dbName}`,
-      [
-        Permission.read("any"),
-        Permission.create("users"),
-        Permission.update("users"),
-        Permission.delete("users"),
-      ]
+    const documentStorage = await tryAwaitWithRetry(
+      async () =>
+        await storage.createBucket(
+          `${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`,
+          `Document Storage ${dbName}`,
+          [
+            Permission.read("any"),
+            Permission.create("users"),
+            Permission.update("users"),
+            Permission.delete("users"),
+          ]
+        )
     );
     return documentStorage;
   }
@@ -102,7 +113,9 @@ export const wipeDocumentStorage = async (
     if (lastFileId) {
       queries.push(Query.cursorAfter(lastFileId));
     }
-    const filesPulled = await storage.listFiles(bucketId, queries);
+    const filesPulled = await tryAwaitWithRetry(
+      async () => await storage.listFiles(bucketId, queries)
+    );
     if (filesPulled.files.length === 0) {
       console.log("No files found, done!");
       moreFiles = false;
@@ -177,7 +190,7 @@ export const backupDatabase = async (
   // Fetch and backup the database details
   let db: Models.Database;
   try {
-    db = await database.get(databaseId);
+    db = await tryAwaitWithRetry(async () => await database.get(databaseId));
   } catch (e) {
     console.error(`Error fetching database: ${e}`);
     await logOperation(
@@ -205,10 +218,13 @@ export const backupDatabase = async (
   let total = 0; // Initialize total to 0, will be updated dynamically
 
   while (moreCollections) {
-    const collectionResponse = await database.listCollections(databaseId, [
-      Query.limit(500), // Adjust the limit as needed
-      ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
-    ]);
+    const collectionResponse = await tryAwaitWithRetry(
+      async () =>
+        await database.listCollections(databaseId, [
+          Query.limit(500), // Adjust the limit as needed
+          ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
+        ])
+    );
 
     total += collectionResponse.collections.length; // Update total with number of collections
 
@@ -218,9 +234,8 @@ export const backupDatabase = async (
     } of collectionResponse.collections) {
       let collectionDocumentCount = 0; // Initialize document count for the current collection
       try {
-        const collection = await database.getCollection(
-          databaseId,
-          collectionId
+        const collection = await tryAwaitWithRetry(
+          async () => await database.getCollection(databaseId, collectionId)
         );
         progress++;
         data.collections.push(JSON.stringify(collection));
@@ -230,13 +245,12 @@ export const backupDatabase = async (
         let moreDocuments = true;
 
         while (moreDocuments) {
-          const documentResponse = await database.listDocuments(
-            databaseId,
-            collectionId,
-            [
-              Query.limit(500), // Adjust the limit as needed
-              ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
-            ]
+          const documentResponse = await tryAwaitWithRetry(
+            async () =>
+              await database.listDocuments(databaseId, collectionId, [
+                Query.limit(500), // Adjust the limit as needed
+                ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
+              ])
           );
 
           total += documentResponse.documents.length; // Update total with number of documents

package/src/migrations/users.ts

@@ -15,6 +15,7 @@ import {
 import _ from "lodash";
 import { logger } from "./logging.js";
 import { splitIntoBatches } from "./migrationHelper.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 
 export class UsersController {
   private config: AppwriteConfig;
@@ -37,49 +38,32 @@ export class UsersController {
   }
 
   async wipeUsers() {
-    const users = await this.users.list([Query.limit(200)]);
-    const allUsers = users.users;
-    let lastDocumentId: string | undefined;
-    if (users.users.length >= 200) {
-      lastDocumentId = users.users[users.users.length - 1].$id;
-    }
-    while (lastDocumentId) {
-      const moreUsers = await this.users.list([
-        Query.limit(200),
-        Query.cursorAfter(lastDocumentId),
-      ]);
-      allUsers.push(...moreUsers.users);
-      if (moreUsers.users.length < 200) {
-        break;
-      }
-      lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
-    }
+    const allUsers = await this.getAllUsers();
     console.log("Deleting all users...");
-    const createBatches = (finalData: any[]) => {
-      let maxBatchLength = 5;
+
+    const createBatches = (finalData: any[], batchSize: number) => {
       const finalBatches: any[][] = [];
-      for (let i = 0; i < finalData.length; i++) {
-        if (i % maxBatchLength === 0) {
-          finalBatches.push([]);
-        }
-        finalBatches[finalBatches.length - 1].push(finalData[i]);
+      for (let i = 0; i < finalData.length; i += batchSize) {
+        finalBatches.push(finalData.slice(i, i + batchSize));
       }
       return finalBatches;
     };
-    // const userPromises: Promise<string>[] = [];
+
     let usersDeleted = 0;
-    for (const user of allUsers) {
-      await this.users.delete(user.$id);
-      usersDeleted++;
+    const batchedUserPromises = createBatches(allUsers, 50); // Batch size of 10
+
+    for (const batch of batchedUserPromises) {
+      console.log(`Deleting ${batch.length} users...`);
+      await Promise.all(
+        batch.map((user) =>
+          tryAwaitWithRetry(async () => await this.users.delete(user.$id))
+        )
+      );
+      usersDeleted += batch.length;
       if (usersDeleted % 100 === 0) {
         console.log(`Deleted ${usersDeleted} users...`);
       }
     }
-    // const batchedUserPromises = createBatches(userPromises);
-    // for (const batch of batchedUserPromises) {
-    //   console.log(`Deleting ${batch.length} users...`);
-    //   await Promise.all(batch);
-    // }
   }
 
   async getAllUsers() {
@@ -92,10 +76,13 @@ export class UsersController {
       let lastDocumentId = users.users[users.users.length - 1].$id;
       allUsers.push(...users.users);
       while (lastDocumentId) {
-        const moreUsers = await this.users.list([
-          Query.limit(200),
-          Query.cursorAfter(lastDocumentId),
-        ]);
+        const moreUsers = await tryAwaitWithRetry(
+          async () =>
+            await this.users.list([
+              Query.limit(200),
+              Query.cursorAfter(lastDocumentId),
+            ])
+        );
         allUsers.push(...moreUsers.users);
         lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
         if (moreUsers.users.length < 200) {
@@ -137,7 +124,10 @@ export class UsersController {
       return user;
     } catch (e) {
       if (e instanceof AppwriteException) {
-        if (e.message.includes("fetch failed")) {
+        if (
+          e.message.toLowerCase().includes("fetch failed") ||
+          e.message.toLowerCase().includes("server error")
+        ) {
           const numberOfAttempts = numAttempts || 0;
           if (numberOfAttempts > 5) {
             throw e;

package/src/utils/helperFunctions.ts

@@ -1,4 +1,4 @@
-import type { Models, Storage } from "node-appwrite";
+import { AppwriteException, type Models, type Storage } from "node-appwrite";
 import fs from "node:fs";
 import path from "node:path";
 import type { CollectionImportData } from "../migrations/dataLoader.js";
@@ -125,3 +125,35 @@ export const finalizeByAttributeMap = async (
     ...item.finalData,
   });
 };
+
+export let numTimesFailedTotal = 0;
+
+/**
+ * Tries to execute the given createFunction and retries up to 5 times if it fails.
+ *
+ * @param {() => Promise<any>} createFunction - The function to be executed.
+ * @param {number} [attemptNum=0] - The number of attempts made so far (default: 0).
+ * @return {Promise<any>} - A promise that resolves to the result of the createFunction or rejects with an error if it fails after 5 attempts.
+ */
+export const tryAwaitWithRetry = async <T>(
+  createFunction: () => Promise<T>,
+  attemptNum: number = 0
+): Promise<T> => {
+  try {
+    return await createFunction();
+  } catch (error) {
+    if (
+      error instanceof AppwriteException &&
+      (error.message.toLowerCase().includes("fetch failed") ||
+        error.message.toLowerCase().includes("server error"))
+    ) {
+      numTimesFailedTotal++;
+      console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
+      if (attemptNum > 5) {
+        throw error;
+      }
+      return tryAwaitWithRetry(createFunction, attemptNum + 1);
+    }
+    throw error;
+  }
+};

package/src/utilsController.ts

@@ -4,7 +4,7 @@ import path from "path";
 import fs from "fs";
 import { load } from "js-yaml";
 import { ImportDataActions } from "./migrations/importDataActions.js";
-import { readFileSync } from "./utils/helperFunctions.js";
+import { numTimesFailedTotal } from "./utils/helperFunctions.js";
 import { afterImportActions } from "./migrations/afterImportActions.js";
 import {
   type AfterImportActions,
@@ -187,6 +187,8 @@ export class UtilsController {
       console.log("Import data complete.");
     }
 
+    console.log("Total number of times Fetch failed: ", numTimesFailedTotal);
+
     // if (options.checkDuplicates) {
     //   await this.checkDuplicates();
     // }