appwrite-utils-cli 0.0.37 → 0.0.38

package/dist/migrations/importController.js

@@ -1,6 +1,6 @@
-import { ID, Query, } from "node-appwrite";
+import { AppwriteException, ID, Query, } from "node-appwrite";
 import _ from "lodash";
-import { areCollectionNamesSame } from "../utils/index.js";
+import { areCollectionNamesSame, tryAwaitWithRetry } from "../utils/index.js";
 import { resolveAndUpdateRelationships } from "./relationships.js";
 import { UsersController } from "./users.js";
 import { logger } from "./logging.js";
@@ -71,7 +71,7 @@ export class ImportController {
                 dataLoader.getCollectionKey(collection.name);
             const importOperationId = dataLoader.collectionImportOperations.get(dataLoader.getCollectionKey(collection.name));
             const createBatches = (finalData) => {
-                let maxBatchLength = 50;
+                let maxBatchLength = 25;
                 const finalBatches = [];
                 for (let i = 0; i < finalData.length; i++) {
                     if (i % maxBatchLength === 0) {
@@ -109,7 +109,7 @@ export class ImportController {
                             .map((item) => {
                             return usersController.createUserAndReturn(item.finalData);
                         });
-                        await Promise.all(userBatchPromises);
+                        const promiseResults = await Promise.allSettled(userBatchPromises);
                         for (const item of batch) {
                             if (item && item.finalData) {
                                 dataLoader.userExistsMap.set(item.finalData.userId ||
@@ -118,26 +118,8 @@ export class ImportController {
                                     item.context.docId, true);
                             }
                         }
-                        console.log("Finished importing users batch", batch.length);
+                        console.log("Finished importing users batch");
                     }
-                    // for (let i = 0; i < usersData.length; i++) {
-                    //   const user = usersData[i];
-                    //   if (user.finalData) {
-                    //     const userId =
-                    //       user.finalData.userId ||
-                    //       user.context.userId ||
-                    //       user.context.docId;
-                    //     if (!dataLoader.userExistsMap.has(userId)) {
-                    //       if (!user.finalData.userId) {
-                    //         user.finalData.userId = userId;
-                    //       }
-                    //       await usersController.createUserAndReturn(user.finalData);
-                    //       dataLoader.userExistsMap.set(userId, true);
-                    //     } else {
-                    //       console.log("Skipped existing user: ", userId);
-                    //     }
-                    //   }
-                    // }
                     console.log("Finished importing users");
                 }
             }
@@ -173,7 +155,7 @@ export class ImportController {
                     if (!item.finalData) {
                         return Promise.resolve();
                     }
-                    return this.database.createDocument(db.$id, collection.$id, id, item.finalData);
+                    return tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, item.finalData));
                 });
                 // Wait for all promises in the current batch to resolve
                 await Promise.allSettled(batchPromises);

package/dist/migrations/importDataActions.d.ts

@@ -20,7 +20,7 @@ export declare class ImportDataActions {
      */
     validateItem(item: any, attributeMap: AttributeMappings, context: {
         [key: string]: any;
-    }): Promise<boolean>;
+    }): boolean;
     executeAfterImportActions(item: any, attributeMap: AttributeMappings, context: {
         [key: string]: any;
     }): Promise<void>;

package/dist/migrations/importDataActions.js

@@ -5,6 +5,7 @@ import { convertObjectBySchema } from "./converters.js";
 import {} from "appwrite-utils";
 import { afterImportActions } from "./afterImportActions.js";
 import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export class ImportDataActions {
     db;
     storage;
@@ -68,7 +69,7 @@ export class ImportDataActions {
      * @param context The context for resolving templated parameters in validation rules.
      * @returns A promise that resolves to true if the item is valid, false otherwise.
      */
-    async validateItem(item, attributeMap, context) {
+    validateItem(item, attributeMap, context) {
         for (const mapping of attributeMap) {
             const { validationActions } = mapping;
             if (!validationActions ||
@@ -111,7 +112,7 @@ export class ImportDataActions {
                 const { action, params } = actionDef;
                 console.log(`Executing post-import action '${action}' for attribute '${mapping.targetKey}' with params ${params.join(", ")}...`);
                 try {
-                    await this.executeAction(action, params, context, item);
+                    await tryAwaitWithRetry(async () => await this.executeAction(action, params, context, item));
                 }
                 catch (error) {
                     logger.error(`Failed to execute post-import action '${action}' for attribute '${mapping.targetKey}':`, error);

package/dist/migrations/indexes.js

@@ -1,5 +1,6 @@
 import { indexSchema } from "appwrite-utils";
 import { Databases, Query } from "node-appwrite";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 // import {}
 export const createOrUpdateIndex = async (dbId, db, collectionId, index) => {
     const existingIndex = await db.listIndexes(dbId, collectionId, [
@@ -13,6 +14,6 @@ export const createOrUpdateIndex = async (dbId, db, collectionId, index) => {
 };
 export const createOrUpdateIndexes = async (dbId, db, collectionId, indexes) => {
     for (const index of indexes) {
-        await createOrUpdateIndex(dbId, db, collectionId, index);
+        await tryAwaitWithRetry(async () => await createOrUpdateIndex(dbId, db, collectionId, index));
     }
 };

package/dist/migrations/migrationHelper.js

@@ -3,6 +3,7 @@ import { BatchSchema, OperationSchema } from "./backup.js";
 import { AttributeMappingsSchema } from "appwrite-utils";
 import { z } from "zod";
 import { logger } from "./logging.js";
+import { tryAwaitWithRetry } from "src/utils/helperFunctions.js";
 /**
  * Object that contains the context for an action that needs to be executed after import
  * Used in the afterImportActionsDefinitions
@@ -90,7 +91,7 @@ export const findOrCreateOperation = async (database, collectionId, operationTyp
     }
 };
 export const updateOperation = async (database, operationId, updateFields) => {
-    await database.updateDocument("migrations", "currentOperations", operationId, updateFields);
+    await tryAwaitWithRetry(async () => await database.updateDocument("migrations", "currentOperations", operationId, updateFields));
 };
 // Actual max 1073741824
 export const maxDataLength = 1073741820;

package/dist/migrations/queue.js

@@ -2,6 +2,7 @@ import { Query } from "node-appwrite";
 import { createOrUpdateAttribute } from "./attributes.js";
 import _ from "lodash";
 import { fetchAndCacheCollectionByName } from "./collections.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const queuedOperations = [];
 export const nameToIdMapping = new Map();
 export const enqueueOperation = (operation) => {
@@ -24,7 +25,7 @@ export const processQueue = async (db, dbId) => {
                 if (operation.collectionId) {
                     console.log(`\tFetching collection by ID: ${operation.collectionId}`);
                     try {
-                        collectionFound = await db.getCollection(dbId, operation.collectionId);
+                        collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
                     }
                     catch (e) {
                         console.log(`\tCollection not found by ID: ${operation.collectionId}`);
@@ -47,7 +48,7 @@ export const processQueue = async (db, dbId) => {
                 // Handle non-relationship operations with a specified collectionId
                 console.log(`\tFetching collection by ID: ${operation.collectionId}`);
                 try {
-                    collectionFound = await db.getCollection(dbId, operation.collectionId);
+                    collectionFound = await tryAwaitWithRetry(async () => await db.getCollection(dbId, operation.collectionId));
                 }
                 catch (e) {
                     console.log(`\tCollection not found by ID: ${operation.collectionId}`);

package/dist/migrations/setupDatabase.js

@@ -2,7 +2,7 @@ import { Databases, ID, Query, Storage } from "node-appwrite";
 import { createOrUpdateAttribute } from "./attributes.js";
 import { createOrUpdateCollections, generateSchemas, wipeDatabase, } from "./collections.js";
 import { getMigrationCollectionSchemas } from "./backup.js";
-import { areCollectionNamesSame, toCamelCase } from "../utils/index.js";
+import { areCollectionNamesSame, toCamelCase, tryAwaitWithRetry, } from "../utils/index.js";
 import { backupDatabase, initOrGetBackupStorage, initOrGetDocumentStorage, wipeDocumentStorage, } from "./storage.js";
 import {} from "appwrite-utils";
 import { nameToIdMapping } from "./queue.js";
@@ -17,17 +17,15 @@ export const setupMigrationDatabase = async (config) => {
     const dbCollections = [];
     const migrationCollectionsSetup = getMigrationCollectionSchemas();
     try {
-        db = await database.get("migrations");
+        db = await tryAwaitWithRetry(async () => await database.get("migrations"));
         console.log("Migrations database found");
     }
     catch (e) {
-        db = await database.create("migrations", "Migrations", true);
+        db = await tryAwaitWithRetry(async () => await database.create("migrations", "Migrations", true));
         console.log("Migrations database created");
     }
     if (db) {
-        const collectionsPulled = await database.listCollections(db.$id, [
-            Query.limit(25),
-        ]);
+        const collectionsPulled = await tryAwaitWithRetry(async () => await database.listCollections(db.$id, [Query.limit(25)]));
         dbCollections.push(...collectionsPulled.collections);
     }
     console.log(`Collections in migrations database: ${dbCollections.length}`);
@@ -36,14 +34,14 @@ export const setupMigrationDatabase = async (config) => {
         const collectionId = toCamelCase(collectionName); // Convert name to toCamelCase for the ID
         let collectionFound = null;
         try {
-            collectionFound = await database.getCollection(db.$id, collectionId);
+            collectionFound = await tryAwaitWithRetry(async () => await database.getCollection(db.$id, collectionId));
         }
         catch (e) {
             console.log(`Collection not found: ${collectionId}`);
         }
         if (!collectionFound) {
             // Create the collection with the provided configuration
-            collectionFound = await database.createCollection(db.$id, collectionId, collectionName, undefined, collection.documentSecurity, collection.enabled);
+            collectionFound = await tryAwaitWithRetry(async () => await database.createCollection(db.$id, collectionId, collectionName, undefined, collection.documentSecurity, collection.enabled));
         }
         for (const attribute of attributes) {
             await createOrUpdateAttribute(database, db.$id, collectionFound, attribute);
@@ -61,10 +59,10 @@ export const ensureDatabasesExist = async (config) => {
         name: "Migrations",
     });
     const dbNames = databasesToEnsure.map((db) => db.name);
-    const existingDatabases = await database.list([Query.equal("name", dbNames)]);
+    const existingDatabases = await tryAwaitWithRetry(async () => await database.list([Query.equal("name", dbNames)]));
     for (const db of databasesToEnsure) {
         if (!existingDatabases.databases.some((d) => d.name === db.name)) {
-            await database.create(db.$id || ID.unique(), db.name, true);
+            await tryAwaitWithRetry(async () => await database.create(db.$id || ID.unique(), db.name, true));
             console.log(`${db.name} database created`);
         }
     }
@@ -73,10 +71,10 @@ export const wipeOtherDatabases = async (database, config) => {
     const databasesToKeep = config.databases.map((db) => db.name.toLowerCase().trim().replace(" ", ""));
     databasesToKeep.push("migrations");
     console.log(`Databases to keep: ${databasesToKeep.join(", ")}`);
-    const allDatabases = await database.list([Query.limit(500)]);
+    const allDatabases = await tryAwaitWithRetry(async () => await database.list([Query.limit(500)]));
     for (const db of allDatabases.databases) {
         if (!databasesToKeep.includes(db.name.toLowerCase().trim().replace(" ", ""))) {
-            await database.delete(db.$id);
+            await tryAwaitWithRetry(async () => await database.delete(db.$id));
             console.log(`Deleted database: ${db.name}`);
         }
     }

package/dist/migrations/storage.js

@@ -1,12 +1,13 @@
 import { Storage, Databases, Query, InputFile, ID, Permission, } from "node-appwrite";
 import {} from "./backup.js";
 import { splitIntoBatches } from "./migrationHelper.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export const logOperation = async (db, dbId, operationDetails, operationId) => {
     try {
         let operation;
         if (operationId) {
             // Update existing operation log
-            operation = await db.updateDocument("migrations", "currentOperations", operationId, operationDetails);
+            operation = await tryAwaitWithRetry(async () => await db.updateDocument("migrations", "currentOperations", operationId, operationDetails));
         }
         else {
             // Create new operation log
@@ -22,29 +23,29 @@ export const logOperation = async (db, dbId, operationDetails, operationId) => {
 };
 export const initOrGetBackupStorage = async (storage) => {
     try {
-        const backupStorage = await storage.getBucket("backupStorage");
+        const backupStorage = await tryAwaitWithRetry(async () => await storage.getBucket("backupStorage"));
         return backupStorage;
     }
     catch (e) {
         // ID backupStorage
         // Name Backups Storage
-        const backupStorage = await storage.createBucket("backupStorage", "Backups Storage");
+        const backupStorage = await tryAwaitWithRetry(async () => await storage.createBucket("backupStorage", "Backups Storage"));
         return backupStorage;
     }
 };
 export const initOrGetDocumentStorage = async (storage, config, dbName) => {
     try {
-        await storage.getBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`);
+        await tryAwaitWithRetry(async () => await storage.getBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`));
     }
     catch (e) {
         // ID documentStorage
         // Name Document Storage
-        const documentStorage = await storage.createBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`, `Document Storage ${dbName}`, [
+        const documentStorage = await tryAwaitWithRetry(async () => await storage.createBucket(`${config.documentBucketId}_${dbName.toLowerCase().replace(" ", "")}`, `Document Storage ${dbName}`, [
             Permission.read("any"),
             Permission.create("users"),
             Permission.update("users"),
             Permission.delete("users"),
-        ]);
+        ]));
         return documentStorage;
     }
 };
@@ -61,7 +62,7 @@ export const wipeDocumentStorage = async (storage, config, dbName) => {
         if (lastFileId) {
             queries.push(Query.cursorAfter(lastFileId));
         }
-        const filesPulled = await storage.listFiles(bucketId, queries);
+        const filesPulled = await tryAwaitWithRetry(async () => await storage.listFiles(bucketId, queries));
         if (filesPulled.files.length === 0) {
             console.log("No files found, done!");
             moreFiles = false;
@@ -124,7 +125,7 @@ export const backupDatabase = async (database, databaseId, storage) => {
     // Fetch and backup the database details
     let db;
     try {
-        db = await database.get(databaseId);
+        db = await tryAwaitWithRetry(async () => await database.get(databaseId));
     }
     catch (e) {
         console.error(`Error fetching database: ${e}`);
@@ -146,25 +147,25 @@ export const backupDatabase = async (database, databaseId, storage) => {
     let progress = 0;
     let total = 0; // Initialize total to 0, will be updated dynamically
     while (moreCollections) {
-        const collectionResponse = await database.listCollections(databaseId, [
+        const collectionResponse = await tryAwaitWithRetry(async () => await database.listCollections(databaseId, [
             Query.limit(500), // Adjust the limit as needed
             ...(lastCollectionId ? [Query.cursorAfter(lastCollectionId)] : []),
-        ]);
+        ]));
         total += collectionResponse.collections.length; // Update total with number of collections
         for (const { $id: collectionId, name: collectionName, } of collectionResponse.collections) {
             let collectionDocumentCount = 0; // Initialize document count for the current collection
             try {
-                const collection = await database.getCollection(databaseId, collectionId);
+                const collection = await tryAwaitWithRetry(async () => await database.getCollection(databaseId, collectionId));
                 progress++;
                 data.collections.push(JSON.stringify(collection));
                 // Initialize pagination for documents within the current collection
                 let lastDocumentId = "";
                 let moreDocuments = true;
                 while (moreDocuments) {
-                    const documentResponse = await database.listDocuments(databaseId, collectionId, [
+                    const documentResponse = await tryAwaitWithRetry(async () => await database.listDocuments(databaseId, collectionId, [
                         Query.limit(500), // Adjust the limit as needed
                         ...(lastDocumentId ? [Query.cursorAfter(lastDocumentId)] : []),
-                    ]);
+                    ]));
                     total += documentResponse.documents.length; // Update total with number of documents
                     collectionDocumentCount += documentResponse.documents.length; // Update document count for the current collection
                     let documentPromises = [];

package/dist/migrations/users.js

@@ -3,6 +3,7 @@ import { AuthUserSchema, } from "../schemas/authUser.js";
 import _ from "lodash";
 import { logger } from "./logging.js";
 import { splitIntoBatches } from "./migrationHelper.js";
+import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export class UsersController {
     config;
     users;
@@ -22,49 +23,25 @@ export class UsersController {
         this.users = new Users(this.config.appwriteClient);
     }
     async wipeUsers() {
-        const users = await this.users.list([Query.limit(200)]);
-        const allUsers = users.users;
-        let lastDocumentId;
-        if (users.users.length >= 200) {
-            lastDocumentId = users.users[users.users.length - 1].$id;
-        }
-        while (lastDocumentId) {
-            const moreUsers = await this.users.list([
-                Query.limit(200),
-                Query.cursorAfter(lastDocumentId),
-            ]);
-            allUsers.push(...moreUsers.users);
-            if (moreUsers.users.length < 200) {
-                break;
-            }
-            lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
-        }
+        const allUsers = await this.getAllUsers();
         console.log("Deleting all users...");
-        const createBatches = (finalData) => {
-            let maxBatchLength = 5;
+        const createBatches = (finalData, batchSize) => {
             const finalBatches = [];
-            for (let i = 0; i < finalData.length; i++) {
-                if (i % maxBatchLength === 0) {
-                    finalBatches.push([]);
-                }
-                finalBatches[finalBatches.length - 1].push(finalData[i]);
+            for (let i = 0; i < finalData.length; i += batchSize) {
+                finalBatches.push(finalData.slice(i, i + batchSize));
             }
             return finalBatches;
         };
-        // const userPromises: Promise<string>[] = [];
         let usersDeleted = 0;
-        for (const user of allUsers) {
-            await this.users.delete(user.$id);
-            usersDeleted++;
+        const batchedUserPromises = createBatches(allUsers, 50); // Batch size of 10
+        for (const batch of batchedUserPromises) {
+            console.log(`Deleting ${batch.length} users...`);
+            await Promise.all(batch.map((user) => tryAwaitWithRetry(async () => await this.users.delete(user.$id))));
+            usersDeleted += batch.length;
             if (usersDeleted % 100 === 0) {
                 console.log(`Deleted ${usersDeleted} users...`);
             }
         }
-        // const batchedUserPromises = createBatches(userPromises);
-        // for (const batch of batchedUserPromises) {
-        //   console.log(`Deleting ${batch.length} users...`);
-        //   await Promise.all(batch);
-        // }
     }
     async getAllUsers() {
         const allUsers = [];
@@ -76,10 +53,10 @@ export class UsersController {
             let lastDocumentId = users.users[users.users.length - 1].$id;
             allUsers.push(...users.users);
             while (lastDocumentId) {
-                const moreUsers = await this.users.list([
+                const moreUsers = await tryAwaitWithRetry(async () => await this.users.list([
                     Query.limit(200),
                     Query.cursorAfter(lastDocumentId),
-                ]);
+                ]));
                 allUsers.push(...moreUsers.users);
                 lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
                 if (moreUsers.users.length < 200) {
@@ -113,7 +90,8 @@ export class UsersController {
         }
         catch (e) {
             if (e instanceof AppwriteException) {
-                if (e.message.includes("fetch failed")) {
+                if (e.message.toLowerCase().includes("fetch failed") ||
+                    e.message.toLowerCase().includes("server error")) {
                     const numberOfAttempts = numAttempts || 0;
                     if (numberOfAttempts > 5) {
                         throw e;

package/dist/utils/helperFunctions.d.ts

@@ -1,4 +1,4 @@
-import type { Models } from "node-appwrite";
+import { type Models } from "node-appwrite";
 import type { CollectionImportData } from "../migrations/dataLoader.js";
 import type { ConfigCollection } from "appwrite-utils";
 export declare const toPascalCase: (str: string) => string;
@@ -35,3 +35,12 @@ export declare const getFileViewUrl: (endpoint: string, projectId: string, bucke
  */
 export declare const getFileDownloadUrl: (endpoint: string, projectId: string, bucketId: string, fileId: string, jwt?: Models.Jwt) => string;
 export declare const finalizeByAttributeMap: (appwriteFolderPath: string, collection: ConfigCollection, item: CollectionImportData["data"][number]) => Promise<any>;
+export declare let numTimesFailedTotal: number;
+/**
+ * Tries to execute the given createFunction and retries up to 5 times if it fails.
+ *
+ * @param {() => Promise<any>} createFunction - The function to be executed.
+ * @param {number} [attemptNum=0] - The number of attempts made so far (default: 0).
+ * @return {Promise<any>} - A promise that resolves to the result of the createFunction or rejects with an error if it fails after 5 attempts.
+ */
+export declare const tryAwaitWithRetry: <T>(createFunction: () => Promise<T>, attemptNum?: number) => Promise<T>;

package/dist/utils/helperFunctions.js

@@ -1,3 +1,4 @@
+import { AppwriteException } from "node-appwrite";
 import fs from "node:fs";
 import path from "node:path";
 export const toPascalCase = (str) => {
@@ -78,3 +79,29 @@ export const finalizeByAttributeMap = async (appwriteFolderPath, collection, ite
         ...item.finalData,
     });
 };
+export let numTimesFailedTotal = 0;
+/**
+ * Tries to execute the given createFunction and retries up to 5 times if it fails.
+ *
+ * @param {() => Promise<any>} createFunction - The function to be executed.
+ * @param {number} [attemptNum=0] - The number of attempts made so far (default: 0).
+ * @return {Promise<any>} - A promise that resolves to the result of the createFunction or rejects with an error if it fails after 5 attempts.
+ */
+export const tryAwaitWithRetry = async (createFunction, attemptNum = 0) => {
+    try {
+        return await createFunction();
+    }
+    catch (error) {
+        if (error instanceof AppwriteException &&
+            (error.message.toLowerCase().includes("fetch failed") ||
+                error.message.toLowerCase().includes("server error"))) {
+            numTimesFailedTotal++;
+            console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
+            if (attemptNum > 5) {
+                throw error;
+            }
+            return tryAwaitWithRetry(createFunction, attemptNum + 1);
+        }
+        throw error;
+    }
+};

package/dist/utilsController.js

@@ -4,7 +4,7 @@ import path from "path";
 import fs from "fs";
 import { load } from "js-yaml";
 import { ImportDataActions } from "./migrations/importDataActions.js";
-import { readFileSync } from "./utils/helperFunctions.js";
+import { numTimesFailedTotal } from "./utils/helperFunctions.js";
 import { afterImportActions } from "./migrations/afterImportActions.js";
 import { converterFunctions, validationRules, AppwriteConfigSchema, } from "appwrite-utils";
 import { ImportController } from "./migrations/importController.js";
@@ -117,6 +117,7 @@ export class UtilsController {
             await importController.run();
             console.log("Import data complete.");
         }
+        console.log("Total number of times Fetch failed: ", numTimesFailedTotal);
         // if (options.checkDuplicates) {
         //   await this.checkDuplicates();
         // }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.37",
+  "version": "0.0.38",
   "main": "src/main.ts",
   "type": "module",
   "repository": {
@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "@types/inquirer": "^9.0.7",
-    "appwrite-utils": "^0.2.5",
+    "appwrite-utils": "^0.2.6",
     "commander": "^12.0.0",
     "inquirer": "^9.2.20",
     "js-yaml": "^4.1.0",