appwrite-utils-cli 0.0.11 → 0.0.12

package/dist/migrations/importController.js

@@ -119,8 +119,8 @@ export class ImportController {
                 continue;
             console.log(`Processing update definitions for collection ID: ${collection.$id}`);
             await this.processBatch(db, collection, importDef, dataToImport);
+            await setAllPendingAfterImportActionsToReady(this.database, db.$id, collection.$id);
         }
-        await setAllPendingAfterImportActionsToReady(this.database, db.$id, collection.$id);
     }
     async loadData(importDef) {
         const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
@@ -151,7 +151,7 @@ export class ImportController {
     async processBatch(db, collection, importDef, dataToImport, updateDefs = [], isMembersCollection = false) {
         for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
             const batch = dataToImport.slice(i, i + this.batchLimit);
-            batch.map(async (item) => {
+            const results = await Promise.allSettled(batch.map(async (item) => {
                 let context = this.createContext(db, collection, item);
                 let finalItem = await this.transformData(item, importDef.attributeMappings);
                 let createIdToUse = undefined;
@@ -191,29 +191,23 @@ export class ImportController {
                     logger.error(`Skipping user & contact creation for ${item} due to lack of email...`);
                 }
                 context = { ...context, ...finalItem };
-                if (!(await this.importDataActions.validateItem(finalItem, importDef.attributeMappings, context))) {
+                const validated = await this.importDataActions.validateItem(finalItem, importDef.attributeMappings, context);
+                if (!validated) {
                     console.error("Validation failed for item:", finalItem);
+                    logger.error("Validation failed for item:", finalItem);
                     return;
                 }
-                let afterContext;
                 if ((importDef.type === "create" || !importDef.type) &&
                     !associatedDoc) {
                     const createdContext = await this.handleCreate(context, finalItem, updateDefs, createIdToUse);
-                    if (createdContext) {
-                        afterContext = createdContext;
-                    }
+                    context = { ...context, ...createdContext };
                     logger.info(`Handled create for ${context.docId}}`);
                 }
                 else {
                     const updatedContext = await this.handleUpdate(context, finalItem, importDef);
-                    if (updatedContext) {
-                        afterContext = updatedContext;
-                    }
+                    context = { ...context, ...updatedContext };
                     logger.info(`Handled update for ${context.docId}`);
                 }
-                if (afterContext) {
-                    context = { ...context, ...afterContext };
-                }
                 const afterImportActionContext = structuredClone(context);
                 const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, finalItem);
                 if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
@@ -232,13 +226,13 @@ export class ImportController {
                     //   attributeMappings: attributeMappingsWithActions,
                     // });
                 }
+            }));
+            results.forEach((result) => {
+                if (result.status === "rejected") {
+                    console.error("A process batch promise was rejected:", result.reason);
+                    logger.error("An error occurred during creation: ", result.reason);
+                }
             });
-            // results.forEach((result) => {
-            //   if (result.status === "rejected") {
-            //     console.error("A process batch promise was rejected:", result.reason);
-            //     logger.error("An error occurred during creation: ", result.reason);
-            //   }
-            // });
         }
     }
     async handleCreate(context, finalItem, updateDefs, id) {
@@ -364,10 +358,6 @@ export class ImportController {
                     await this.importDataActions.executeAfterImportActions(finalItem, attributeMappings, context);
                     // Mark batch as processed
                     await this.database.deleteDocument("migrations", "batches", batch.$id);
-                    await updateOperation(this.database, operation.$id, {
-                        status: "completed",
-                        batches: [],
-                    });
                 }
                 catch (error) {
                     logger.error(`Failed to execute batch ${batch.$id}:`, error);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -113,7 +113,6 @@ export class ImportController {
   async importCollections(db: ConfigDatabase) {
     const maxParallel = 3; // Maximum number of collections to process in parallel
     let activePromises: Promise<void>[] = []; // Array to keep track of active promises
-
     for (const collection of this.config.collections) {
       // Function that returns a promise for processing a single collection
       const processCollection = async (col: ConfigCollection) => {
@@ -200,13 +199,12 @@ export class ImportController {
         `Processing update definitions for collection ID: ${collection.$id}`
       );
       await this.processBatch(db, collection, importDef, dataToImport);
+      await setAllPendingAfterImportActionsToReady(
+        this.database,
+        db.$id,
+        collection.$id
+      );
     }
-
-    await setAllPendingAfterImportActionsToReady(
-      this.database,
-      db.$id,
-      collection.$id
-    );
   }
 
   async loadData(importDef: ImportDef): Promise<any[]> {
@@ -258,145 +256,138 @@ export class ImportController {
   ) {
     for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
       const batch = dataToImport.slice(i, i + this.batchLimit);
-      batch.map(async (item: any) => {
-        let context = this.createContext(db, collection, item);
-        let finalItem = await this.transformData(
-          item,
-          importDef.attributeMappings
-        );
-        let createIdToUse: string | undefined = undefined;
-        let associatedDoc: Models.Document | undefined;
-        if (
-          isMembersCollection &&
-          (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
-        ) {
-          console.log("Found members collection, creating user...");
-          const usersController = new UsersController(
-            this.config,
-            this.database
-          );
-          const userToCreate = AuthUserCreateSchema.safeParse({
-            ...finalItem,
-          });
-          if (!userToCreate.success) {
-            console.error(userToCreate.error);
-            logger.error(userToCreate.error);
-            return;
-          }
-          const user = await usersController.createUserAndReturn(
-            userToCreate.data
-          );
-          createIdToUse = user.$id;
-          context.docId = createIdToUse;
-          context = { ...context, ...user };
-          console.log(
-            "Created user, deleting keys in finalItem that exist in user..."
-          );
-          const associatedDocFound = await this.database.listDocuments(
-            db.$id,
-            context.collId,
-            [Query.equal("$id", createIdToUse)]
+      const results = await Promise.allSettled(
+        batch.map(async (item: any) => {
+          let context = this.createContext(db, collection, item);
+          let finalItem = await this.transformData(
+            item,
+            importDef.attributeMappings
           );
-          if (associatedDocFound.documents.length > 0) {
-            associatedDoc = associatedDocFound.documents[0];
-          }
-          // Delete keys in finalItem that also exist in user
-          let deletedKeys: string[] = [];
-          Object.keys(finalItem).forEach((key) => {
-            if (user.hasOwnProperty(key)) {
-              delete finalItem[key];
-              deletedKeys.push(key);
+          let createIdToUse: string | undefined = undefined;
+          let associatedDoc: Models.Document | undefined;
+          if (
+            isMembersCollection &&
+            (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
+          ) {
+            console.log("Found members collection, creating user...");
+            const usersController = new UsersController(
+              this.config,
+              this.database
+            );
+            const userToCreate = AuthUserCreateSchema.safeParse({
+              ...finalItem,
+            });
+            if (!userToCreate.success) {
+              console.error(userToCreate.error);
+              logger.error(userToCreate.error);
+              return;
             }
-          });
-          console.log(
-            `Set createIdToUse to ${createIdToUse}. Deleted keys: ${deletedKeys.join(
-              ", "
-            )}.`
-          );
-        } else if (isMembersCollection) {
-          logger.error(
-            `Skipping user & contact creation for ${item} due to lack of email...`
-          );
-        }
-
-        context = { ...context, ...finalItem };
+            const user = await usersController.createUserAndReturn(
+              userToCreate.data
+            );
+            createIdToUse = user.$id;
+            context.docId = createIdToUse;
+            context = { ...context, ...user };
+            console.log(
+              "Created user, deleting keys in finalItem that exist in user..."
+            );
+            const associatedDocFound = await this.database.listDocuments(
+              db.$id,
+              context.collId,
+              [Query.equal("$id", createIdToUse)]
+            );
+            if (associatedDocFound.documents.length > 0) {
+              associatedDoc = associatedDocFound.documents[0];
+            }
+            // Delete keys in finalItem that also exist in user
+            let deletedKeys: string[] = [];
+            Object.keys(finalItem).forEach((key) => {
+              if (user.hasOwnProperty(key)) {
+                delete finalItem[key];
+                deletedKeys.push(key);
+              }
+            });
+            console.log(
+              `Set createIdToUse to ${createIdToUse}. Deleted keys: ${deletedKeys.join(
+                ", "
+              )}.`
+            );
+          } else if (isMembersCollection) {
+            logger.error(
+              `Skipping user & contact creation for ${item} due to lack of email...`
+            );
+          }
 
-        if (
-          !(await this.importDataActions.validateItem(
+          context = { ...context, ...finalItem };
+          const validated = await this.importDataActions.validateItem(
             finalItem,
             importDef.attributeMappings,
             context
-          ))
-        ) {
-          console.error("Validation failed for item:", finalItem);
-          return;
-        }
-
-        let afterContext;
-        if (
-          (importDef.type === "create" || !importDef.type) &&
-          !associatedDoc
-        ) {
-          const createdContext = await this.handleCreate(
-            context,
-            finalItem,
-            updateDefs,
-            createIdToUse
           );
-          if (createdContext) {
-            afterContext = createdContext;
+          if (!validated) {
+            console.error("Validation failed for item:", finalItem);
+            logger.error("Validation failed for item:", finalItem);
+            return;
           }
-          logger.info(`Handled create for ${context.docId}}`);
-        } else {
-          const updatedContext = await this.handleUpdate(
-            context,
-            finalItem,
-            importDef
-          );
-          if (updatedContext) {
-            afterContext = updatedContext;
+
+          if (
+            (importDef.type === "create" || !importDef.type) &&
+            !associatedDoc
+          ) {
+            const createdContext = await this.handleCreate(
+              context,
+              finalItem,
+              updateDefs,
+              createIdToUse
+            );
+            context = { ...context, ...createdContext };
+            logger.info(`Handled create for ${context.docId}}`);
+          } else {
+            const updatedContext = await this.handleUpdate(
+              context,
+              finalItem,
+              importDef
+            );
+            context = { ...context, ...updatedContext };
+            logger.info(`Handled update for ${context.docId}`);
           }
-          logger.info(`Handled update for ${context.docId}`);
-        }
-        if (afterContext) {
-          context = { ...context, ...afterContext };
-        }
-        const afterImportActionContext = structuredClone(context);
-        const attributeMappingsWithActions =
-          this.getAttributeMappingsWithActions(
-            importDef.attributeMappings,
-            context,
-            finalItem
-          );
-        if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-          logger.info(
-            `Pushing to post-import actions queue for ${context.docId}`
-          );
-          const afterImportOperationContext = ContextObject.parse({
-            dbId: db.$id,
-            collectionId: collection.$id,
-            finalItem: finalItem,
-            attributeMappings: attributeMappingsWithActions,
-            context: afterImportActionContext,
-          });
-          await createOrFindAfterImportOperation(
-            this.database,
-            context.collId,
-            afterImportOperationContext
-          );
-          // this.postImportActionsQueue.push({
-          //   context: afterImportActionContext,
-          //   finalItem: finalItem,
-          //   attributeMappings: attributeMappingsWithActions,
-          // });
+          const afterImportActionContext = structuredClone(context);
+          const attributeMappingsWithActions =
+            this.getAttributeMappingsWithActions(
+              importDef.attributeMappings,
+              context,
+              finalItem
+            );
+          if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
+            logger.info(
+              `Pushing to post-import actions queue for ${context.docId}`
+            );
+            const afterImportOperationContext = ContextObject.parse({
+              dbId: db.$id,
+              collectionId: collection.$id,
+              finalItem: finalItem,
+              attributeMappings: attributeMappingsWithActions,
+              context: afterImportActionContext,
+            });
+            await createOrFindAfterImportOperation(
+              this.database,
+              context.collId,
+              afterImportOperationContext
+            );
+            // this.postImportActionsQueue.push({
+            //   context: afterImportActionContext,
+            //   finalItem: finalItem,
+            //   attributeMappings: attributeMappingsWithActions,
+            // });
+          }
+        })
+      );
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          console.error("A process batch promise was rejected:", result.reason);
+          logger.error("An error occurred during creation: ", result.reason);
         }
       });
-      // results.forEach((result) => {
-      //   if (result.status === "rejected") {
-      //     console.error("A process batch promise was rejected:", result.reason);
-      //     logger.error("An error occurred during creation: ", result.reason);
-      //   }
-      // });
     }
   }
 
@@ -584,10 +575,6 @@ export class ImportController {
             "batches",
             batch.$id
           );
-          await updateOperation(this.database, operation.$id, {
-            status: "completed",
-            batches: [],
-          });
         } catch (error) {
           logger.error(`Failed to execute batch ${batch.$id}:`, error);
         }