appwrite-utils-cli 0.0.19 → 0.0.21

package/dist/migrations/importController.js

@@ -59,6 +59,7 @@ export class ImportController {
             await this.importCollections(db);
             await resolveAndUpdateRelationships(db.$id, this.database, this.config);
             await this.executePostImportActions(db.$id);
+            await this.executePostImportActions(db.$id);
             console.log(`---------------------------------`);
             console.log(`Finished import data for database: ${db.name}`);
             console.log(`---------------------------------`);
@@ -147,7 +148,7 @@ export class ImportController {
     async processBatch(db, collection, importDef, dataToImport, updateDefs = [], isMembersCollection = false) {
         for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
             const batch = dataToImport.slice(i, i + this.batchLimit);
-            const results = await Promise.allSettled(batch.map(async (item) => {
+            for (const item of batch) {
                 let context = this.createContext(db, collection, item);
                 let finalItem = await this.transformData(item, importDef.attributeMappings);
                 let createIdToUse = undefined;
@@ -207,31 +208,31 @@ export class ImportController {
                 const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, afterImportActionContext, finalItem);
                 if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
                     logger.info(`Pushing to post-import actions queue for ${context.docId}`);
-                    // const afterImportOperationContext = ContextObject.parse({
-                    //   dbId: db.$id,
-                    //   collectionId: collection.$id,
-                    //   finalItem: finalItem,
-                    //   attributeMappings: attributeMappingsWithActions,
-                    //   context: afterImportActionContext,
-                    // });
-                    // await createOrFindAfterImportOperation(
-                    //   this.database,
-                    //   context.collId,
-                    //   afterImportOperationContext
-                    // );
-                    this.postImportActionsQueue.push({
-                        context: afterImportActionContext,
+                    const afterImportOperationContext = ContextObject.parse({
+                        dbId: db.$id,
+                        collectionId: collection.$id,
                         finalItem: finalItem,
                         attributeMappings: attributeMappingsWithActions,
+                        context: afterImportActionContext,
                     });
+                    await createOrFindAfterImportOperation(this.database, context.collId, afterImportOperationContext);
+                    // this.postImportActionsQueue.push({
+                    //   context: afterImportActionContext,
+                    //   finalItem: finalItem,
+                    //   attributeMappings: attributeMappingsWithActions,
+                    // });
                 }
-            }));
-            results.forEach((result) => {
-                if (result.status === "rejected") {
-                    console.error("A process batch promise was rejected:", result.reason);
-                    logger.error("An error occurred during creation: ", result.reason);
-                }
-            });
+            }
+            // const results = await Promise.allSettled(
+            //   batch.map(async (item: any) => {
+            //   })
+            // );
+            // results.forEach((result) => {
+            //   if (result.status === "rejected") {
+            //     console.error("A process batch promise was rejected:", result.reason);
+            //     logger.error("An error occurred during creation: ", result.reason);
+            //   }
+            // });
         }
     }
     async handleCreate(context, finalItem, updateDefs, id) {
@@ -318,33 +319,41 @@ export class ImportController {
         });
     }
     async executePostImportActions(dbId) {
-        let actionQueue = [];
-        for (const action of this.postImportActionsQueue) {
-            actionQueue.push(this.importDataActions.executeAfterImportActions(action.finalItem, action.attributeMappings, action.context));
-        }
-        const BATCH_LIMIT = 20;
-        const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
-        for (const queue of splitQueue) {
-            const results = await Promise.allSettled(queue);
-            results.forEach((result) => {
-                if (result.status === "rejected") {
-                    console.error("An action promise was rejected:", result.reason);
-                    logger.error(`An action promise was rejected: ${result.reason} -- ${JSON.stringify(result)}`);
-                }
-            });
-        }
-        // const collectionActionsPromises = [];
-        // for (const collection of this.config.collections) {
-        //   collectionActionsPromises.push(
-        //     this.executeActionsInParallel(dbId, collection)
+        // let actionQueue: Promise<any>[] = [];
+        // for (const action of this.postImportActionsQueue) {
+        //   actionQueue.push(
+        //     this.importDataActions.executeAfterImportActions(
+        //       action.finalItem,
+        //       action.attributeMappings,
+        //       action.context
+        //     )
         //   );
         // }
-        // const results = await Promise.allSettled(collectionActionsPromises);
-        // results.forEach((result) => {
-        //   if (result.status === "rejected") {
-        //     console.error("A process batch promise was rejected:", result.reason);
-        //   }
-        // });
+        // const BATCH_LIMIT = 5;
+        // const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
+        // for (const queue of splitQueue) {
+        //   const results = await Promise.allSettled(queue);
+        //   results.forEach((result) => {
+        //     if (result.status === "rejected") {
+        //       console.error("An action promise was rejected:", result.reason);
+        //       logger.error(
+        //         `An action promise was rejected: ${
+        //           result.reason
+        //         } -- ${JSON.stringify(result)}`
+        //       );
+        //     }
+        //   });
+        // }
+        const collectionActionsPromises = [];
+        for (const collection of this.config.collections) {
+            collectionActionsPromises.push(this.executeActionsInParallel(dbId, collection));
+        }
+        const results = await Promise.allSettled(collectionActionsPromises);
+        results.forEach((result) => {
+            if (result.status === "rejected") {
+                console.error("A process batch promise was rejected:", result.reason);
+            }
+        });
     }
     async executeActionsInParallel(dbId, collection) {
         const collectionExists = await checkForCollection(this.database, dbId, collection);

package/dist/migrations/migrationHelper.js

@@ -49,7 +49,6 @@ export const getAfterImportOperations = async (database, collectionId) => {
         const query = [
             Query.equal("collectionId", collectionId),
             Query.equal("operationType", "afterImportAction"),
-            Query.equal("status", "ready"),
             Query.limit(100),
         ];
         if (lastDocumentId) {
@@ -58,7 +57,7 @@ export const getAfterImportOperations = async (database, collectionId) => {
         const operations = await database.listDocuments("migrations", "currentOperations", query);
         total = operations.total; // Update total with the latest fetch
         allOperations.push(...operations.documents);
-        if (operations.documents.length > 0) {
+        if (operations.documents.length > 0 && operations.documents.length >= 100) {
             lastDocumentId =
                 operations.documents[operations.documents.length - 1].$id;
         }
@@ -69,11 +68,7 @@ export const getAfterImportOperations = async (database, collectionId) => {
 export const setAllPendingAfterImportActionsToReady = async (database, dbId, collectionId) => {
     let lastDocumentId;
     do {
-        const query = [
-            Query.equal("collectionId", collectionId),
-            Query.equal("status", "pending"),
-            Query.limit(100),
-        ];
+        const query = [Query.equal("collectionId", collectionId), Query.limit(100)];
         if (lastDocumentId) {
             query.push(Query.cursorAfter(lastDocumentId));
         }
@@ -83,7 +78,7 @@ export const setAllPendingAfterImportActionsToReady = async (database, dbId, col
             await database.updateDocument("migrations", "currentOperations", operation.$id, { status: "ready" });
         }
         // Prepare for the next iteration in case there are more than 100 documents
-        if (operations.documents.length > 0) {
+        if (operations.documents.length > 0 && operations.documents.length >= 100) {
             lastDocumentId =
                 operations.documents[operations.documents.length - 1].$id;
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -104,6 +104,7 @@ export class ImportController {
       await this.importCollections(db);
       await resolveAndUpdateRelationships(db.$id, this.database!, this.config!);
       await this.executePostImportActions(db.$id);
+      await this.executePostImportActions(db.$id);
       console.log(`---------------------------------`);
       console.log(`Finished import data for database: ${db.name}`);
       console.log(`---------------------------------`);
@@ -256,133 +257,135 @@ export class ImportController {
   ) {
     for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
       const batch = dataToImport.slice(i, i + this.batchLimit);
-      const results = await Promise.allSettled(
-        batch.map(async (item: any) => {
-          let context = this.createContext(db, collection, item);
-          let finalItem = await this.transformData(
-            item,
-            importDef.attributeMappings
+      for (const item of batch) {
+        let context = this.createContext(db, collection, item);
+        let finalItem = await this.transformData(
+          item,
+          importDef.attributeMappings
+        );
+        let createIdToUse: string | undefined = undefined;
+        let associatedDoc: Models.Document | undefined;
+        if (
+          isMembersCollection &&
+          (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
+        ) {
+          const usersController = new UsersController(
+            this.config,
+            this.database
           );
-          let createIdToUse: string | undefined = undefined;
-          let associatedDoc: Models.Document | undefined;
-          if (
-            isMembersCollection &&
-            (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
-          ) {
-            const usersController = new UsersController(
-              this.config,
-              this.database
-            );
-            const userToCreate = AuthUserCreateSchema.safeParse({
-              ...finalItem,
-            });
-            if (!userToCreate.success) {
-              console.error(userToCreate.error);
-              logger.error(userToCreate.error);
-              return;
-            }
-            const user = await usersController.createUserAndReturn(
-              userToCreate.data
-            );
-            if (!user) {
-              logger.error(
-                `Skipping user & contact creation for ${item} because of an error...`
-              );
-              return;
-            }
-            createIdToUse = user.$id;
-            context.docId = createIdToUse;
-            context = { ...context, ...user };
-            const associatedDocFound = await this.database.listDocuments(
-              db.$id,
-              context.collId,
-              [Query.equal("$id", createIdToUse)]
-            );
-            if (associatedDocFound.documents.length > 0) {
-              associatedDoc = associatedDocFound.documents[0];
-            }
-            // Delete keys in finalItem that also exist in user
-            let deletedKeys: string[] = [];
-            Object.keys(finalItem).forEach((key) => {
-              if (user.hasOwnProperty(key)) {
-                delete finalItem[key];
-                deletedKeys.push(key);
-              }
-            });
-          } else if (isMembersCollection) {
+          const userToCreate = AuthUserCreateSchema.safeParse({
+            ...finalItem,
+          });
+          if (!userToCreate.success) {
+            console.error(userToCreate.error);
+            logger.error(userToCreate.error);
+            return;
+          }
+          const user = await usersController.createUserAndReturn(
+            userToCreate.data
+          );
+          if (!user) {
             logger.error(
-              `Skipping user & contact creation for ${item} due to lack of email...`
+              `Skipping user & contact creation for ${item} because of an error...`
             );
+            return;
           }
+          createIdToUse = user.$id;
+          context.docId = createIdToUse;
+          context = { ...context, ...user };
+          const associatedDocFound = await this.database.listDocuments(
+            db.$id,
+            context.collId,
+            [Query.equal("$id", createIdToUse)]
+          );
+          if (associatedDocFound.documents.length > 0) {
+            associatedDoc = associatedDocFound.documents[0];
+          }
+          // Delete keys in finalItem that also exist in user
+          let deletedKeys: string[] = [];
+          Object.keys(finalItem).forEach((key) => {
+            if (user.hasOwnProperty(key)) {
+              delete finalItem[key];
+              deletedKeys.push(key);
+            }
+          });
+        } else if (isMembersCollection) {
+          logger.error(
+            `Skipping user & contact creation for ${item} due to lack of email...`
+          );
+        }
+
+        context = { ...context, ...finalItem };
+        const validated = await this.importDataActions.validateItem(
+          finalItem,
+          importDef.attributeMappings,
+          context
+        );
+        if (!validated) {
+          console.error("Validation failed for item:", finalItem);
+          logger.error("Validation failed for item:", finalItem);
+          return;
+        }
 
-          context = { ...context, ...finalItem };
-          const validated = await this.importDataActions.validateItem(
+        if (
+          (importDef.type === "create" || !importDef.type) &&
+          !associatedDoc
+        ) {
+          const createdContext = await this.handleCreate(
+            context,
+            finalItem,
+            updateDefs,
+            createIdToUse
+          );
+          context = { ...context, ...createdContext };
+        } else {
+          const updatedContext = await this.handleUpdate(
+            context,
             finalItem,
+            importDef
+          );
+          context = { ...context, ...updatedContext };
+        }
+        const afterImportActionContext = structuredClone(context);
+        const attributeMappingsWithActions =
+          this.getAttributeMappingsWithActions(
             importDef.attributeMappings,
-            context
+            afterImportActionContext,
+            finalItem
           );
-          if (!validated) {
-            console.error("Validation failed for item:", finalItem);
-            logger.error("Validation failed for item:", finalItem);
-            return;
-          }
-
-          if (
-            (importDef.type === "create" || !importDef.type) &&
-            !associatedDoc
-          ) {
-            const createdContext = await this.handleCreate(
-              context,
-              finalItem,
-              updateDefs,
-              createIdToUse
-            );
-            context = { ...context, ...createdContext };
-          } else {
-            const updatedContext = await this.handleUpdate(
-              context,
-              finalItem,
-              importDef
-            );
-            context = { ...context, ...updatedContext };
-          }
-          const afterImportActionContext = structuredClone(context);
-          const attributeMappingsWithActions =
-            this.getAttributeMappingsWithActions(
-              importDef.attributeMappings,
-              afterImportActionContext,
-              finalItem
-            );
-          if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-            logger.info(
-              `Pushing to post-import actions queue for ${context.docId}`
-            );
-            // const afterImportOperationContext = ContextObject.parse({
-            //   dbId: db.$id,
-            //   collectionId: collection.$id,
-            //   finalItem: finalItem,
-            //   attributeMappings: attributeMappingsWithActions,
-            //   context: afterImportActionContext,
-            // });
-            // await createOrFindAfterImportOperation(
-            //   this.database,
-            //   context.collId,
-            //   afterImportOperationContext
-            // );
-            this.postImportActionsQueue.push({
-              context: afterImportActionContext,
-              finalItem: finalItem,
-              attributeMappings: attributeMappingsWithActions,
-            });
-          }
-        })
-      );
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          console.error("A process batch promise was rejected:", result.reason);
-          logger.error("An error occurred during creation: ", result.reason);
+        if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
+          logger.info(
+            `Pushing to post-import actions queue for ${context.docId}`
+          );
+          const afterImportOperationContext = ContextObject.parse({
+            dbId: db.$id,
+            collectionId: collection.$id,
+            finalItem: finalItem,
+            attributeMappings: attributeMappingsWithActions,
+            context: afterImportActionContext,
+          });
+          await createOrFindAfterImportOperation(
+            this.database,
+            context.collId,
+            afterImportOperationContext
+          );
+          // this.postImportActionsQueue.push({
+          //   context: afterImportActionContext,
+          //   finalItem: finalItem,
+          //   attributeMappings: attributeMappingsWithActions,
+          // });
         }
-      });
+      }
+      // const results = await Promise.allSettled(
+      //   batch.map(async (item: any) => {
+      //   })
+      // );
+      // results.forEach((result) => {
+      //   if (result.status === "rejected") {
+      //     console.error("A process batch promise was rejected:", result.reason);
+      //     logger.error("An error occurred during creation: ", result.reason);
+      //   }
+      // });
     }
   }
 
@@ -512,43 +515,43 @@ export class ImportController {
   }
 
   async executePostImportActions(dbId: string) {
-    let actionQueue: Promise<any>[] = [];
-    for (const action of this.postImportActionsQueue) {
-      actionQueue.push(
-        this.importDataActions.executeAfterImportActions(
-          action.finalItem,
-          action.attributeMappings,
-          action.context
-        )
-      );
-    }
-    const BATCH_LIMIT = 20;
-    const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
-    for (const queue of splitQueue) {
-      const results = await Promise.allSettled(queue);
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          console.error("An action promise was rejected:", result.reason);
-          logger.error(
-            `An action promise was rejected: ${
-              result.reason
-            } -- ${JSON.stringify(result)}`
-          );
-        }
-      });
-    }
-    // const collectionActionsPromises = [];
-    // for (const collection of this.config.collections) {
-    //   collectionActionsPromises.push(
-    //     this.executeActionsInParallel(dbId, collection)
+    // let actionQueue: Promise<any>[] = [];
+    // for (const action of this.postImportActionsQueue) {
+    //   actionQueue.push(
+    //     this.importDataActions.executeAfterImportActions(
+    //       action.finalItem,
+    //       action.attributeMappings,
+    //       action.context
+    //     )
     //   );
     // }
-    // const results = await Promise.allSettled(collectionActionsPromises);
-    // results.forEach((result) => {
-    //   if (result.status === "rejected") {
-    //     console.error("A process batch promise was rejected:", result.reason);
-    //   }
-    // });
+    // const BATCH_LIMIT = 5;
+    // const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
+    // for (const queue of splitQueue) {
+    //   const results = await Promise.allSettled(queue);
+    //   results.forEach((result) => {
+    //     if (result.status === "rejected") {
+    //       console.error("An action promise was rejected:", result.reason);
+    //       logger.error(
+    //         `An action promise was rejected: ${
+    //           result.reason
+    //         } -- ${JSON.stringify(result)}`
+    //       );
+    //     }
+    //   });
+    // }
+    const collectionActionsPromises = [];
+    for (const collection of this.config.collections) {
+      collectionActionsPromises.push(
+        this.executeActionsInParallel(dbId, collection)
+      );
+    }
+    const results = await Promise.allSettled(collectionActionsPromises);
+    results.forEach((result) => {
+      if (result.status === "rejected") {
+        console.error("A process batch promise was rejected:", result.reason);
+      }
+    });
   }
 
   async executeActionsInParallel(dbId: string, collection: ConfigCollection) {

package/src/migrations/migrationHelper.ts

@@ -78,7 +78,6 @@ export const getAfterImportOperations = async (
     const query = [
       Query.equal("collectionId", collectionId),
       Query.equal("operationType", "afterImportAction"),
-      Query.equal("status", "ready"),
       Query.limit(100),
     ];
 
@@ -94,7 +93,7 @@ export const getAfterImportOperations = async (
     total = operations.total; // Update total with the latest fetch
     allOperations.push(...operations.documents);
 
-    if (operations.documents.length > 0) {
+    if (operations.documents.length > 0 && operations.documents.length >= 100) {
       lastDocumentId =
         operations.documents[operations.documents.length - 1].$id;
     }
@@ -111,11 +110,7 @@ export const setAllPendingAfterImportActionsToReady = async (
 ) => {
   let lastDocumentId: string | undefined;
   do {
-    const query = [
-      Query.equal("collectionId", collectionId),
-      Query.equal("status", "pending"),
-      Query.limit(100),
-    ];
+    const query = [Query.equal("collectionId", collectionId), Query.limit(100)];
 
     if (lastDocumentId) {
       query.push(Query.cursorAfter(lastDocumentId));
@@ -138,7 +133,7 @@ export const setAllPendingAfterImportActionsToReady = async (
     }
 
     // Prepare for the next iteration in case there are more than 100 documents
-    if (operations.documents.length > 0) {
+    if (operations.documents.length > 0 && operations.documents.length >= 100) {
       lastDocumentId =
         operations.documents[operations.documents.length - 1].$id;
     } else {