appwrite-utils-cli 0.0.20 → 0.0.21

package/dist/migrations/importController.js

@@ -148,7 +148,7 @@ export class ImportController {
     async processBatch(db, collection, importDef, dataToImport, updateDefs = [], isMembersCollection = false) {
         for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
             const batch = dataToImport.slice(i, i + this.batchLimit);
-            const results = await Promise.allSettled(batch.map(async (item) => {
+            for (const item of batch) {
                 let context = this.createContext(db, collection, item);
                 let finalItem = await this.transformData(item, importDef.attributeMappings);
                 let createIdToUse = undefined;
@@ -222,13 +222,17 @@ export class ImportController {
                     //   attributeMappings: attributeMappingsWithActions,
                     // });
                 }
-            }));
-            results.forEach((result) => {
-                if (result.status === "rejected") {
-                    console.error("A process batch promise was rejected:", result.reason);
-                    logger.error("An error occurred during creation: ", result.reason);
-                }
-            });
+            }
+            // const results = await Promise.allSettled(
+            //   batch.map(async (item: any) => {
+            //   })
+            // );
+            // results.forEach((result) => {
+            //   if (result.status === "rejected") {
+            //     console.error("A process batch promise was rejected:", result.reason);
+            //     logger.error("An error occurred during creation: ", result.reason);
+            //   }
+            // });
         }
     }
     async handleCreate(context, finalItem, updateDefs, id) {

package/dist/migrations/migrationHelper.js

@@ -49,7 +49,6 @@ export const getAfterImportOperations = async (database, collectionId) => {
         const query = [
             Query.equal("collectionId", collectionId),
             Query.equal("operationType", "afterImportAction"),
-            Query.equal("status", "ready"),
             Query.limit(100),
         ];
         if (lastDocumentId) {
@@ -58,7 +57,7 @@ export const getAfterImportOperations = async (database, collectionId) => {
         const operations = await database.listDocuments("migrations", "currentOperations", query);
         total = operations.total; // Update total with the latest fetch
         allOperations.push(...operations.documents);
-        if (operations.documents.length > 0) {
+        if (operations.documents.length > 0 && operations.documents.length >= 100) {
             lastDocumentId =
                 operations.documents[operations.documents.length - 1].$id;
         }
@@ -79,7 +78,7 @@ export const setAllPendingAfterImportActionsToReady = async (database, dbId, col
             await database.updateDocument("migrations", "currentOperations", operation.$id, { status: "ready" });
         }
         // Prepare for the next iteration in case there are more than 100 documents
-        if (operations.documents.length > 0) {
+        if (operations.documents.length > 0 && operations.documents.length >= 100) {
             lastDocumentId =
                 operations.documents[operations.documents.length - 1].$id;
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.20",
+  "version": "0.0.21",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -257,133 +257,135 @@ export class ImportController {
   ) {
     for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
       const batch = dataToImport.slice(i, i + this.batchLimit);
-      const results = await Promise.allSettled(
-        batch.map(async (item: any) => {
-          let context = this.createContext(db, collection, item);
-          let finalItem = await this.transformData(
-            item,
-            importDef.attributeMappings
+      for (const item of batch) {
+        let context = this.createContext(db, collection, item);
+        let finalItem = await this.transformData(
+          item,
+          importDef.attributeMappings
+        );
+        let createIdToUse: string | undefined = undefined;
+        let associatedDoc: Models.Document | undefined;
+        if (
+          isMembersCollection &&
+          (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
+        ) {
+          const usersController = new UsersController(
+            this.config,
+            this.database
           );
-          let createIdToUse: string | undefined = undefined;
-          let associatedDoc: Models.Document | undefined;
-          if (
-            isMembersCollection &&
-            (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
-          ) {
-            const usersController = new UsersController(
-              this.config,
-              this.database
-            );
-            const userToCreate = AuthUserCreateSchema.safeParse({
-              ...finalItem,
-            });
-            if (!userToCreate.success) {
-              console.error(userToCreate.error);
-              logger.error(userToCreate.error);
-              return;
-            }
-            const user = await usersController.createUserAndReturn(
-              userToCreate.data
-            );
-            if (!user) {
-              logger.error(
-                `Skipping user & contact creation for ${item} because of an error...`
-              );
-              return;
-            }
-            createIdToUse = user.$id;
-            context.docId = createIdToUse;
-            context = { ...context, ...user };
-            const associatedDocFound = await this.database.listDocuments(
-              db.$id,
-              context.collId,
-              [Query.equal("$id", createIdToUse)]
-            );
-            if (associatedDocFound.documents.length > 0) {
-              associatedDoc = associatedDocFound.documents[0];
-            }
-            // Delete keys in finalItem that also exist in user
-            let deletedKeys: string[] = [];
-            Object.keys(finalItem).forEach((key) => {
-              if (user.hasOwnProperty(key)) {
-                delete finalItem[key];
-                deletedKeys.push(key);
-              }
-            });
-          } else if (isMembersCollection) {
+          const userToCreate = AuthUserCreateSchema.safeParse({
+            ...finalItem,
+          });
+          if (!userToCreate.success) {
+            console.error(userToCreate.error);
+            logger.error(userToCreate.error);
+            return;
+          }
+          const user = await usersController.createUserAndReturn(
+            userToCreate.data
+          );
+          if (!user) {
             logger.error(
-              `Skipping user & contact creation for ${item} due to lack of email...`
+              `Skipping user & contact creation for ${item} because of an error...`
             );
+            return;
+          }
+          createIdToUse = user.$id;
+          context.docId = createIdToUse;
+          context = { ...context, ...user };
+          const associatedDocFound = await this.database.listDocuments(
+            db.$id,
+            context.collId,
+            [Query.equal("$id", createIdToUse)]
+          );
+          if (associatedDocFound.documents.length > 0) {
+            associatedDoc = associatedDocFound.documents[0];
           }
+          // Delete keys in finalItem that also exist in user
+          let deletedKeys: string[] = [];
+          Object.keys(finalItem).forEach((key) => {
+            if (user.hasOwnProperty(key)) {
+              delete finalItem[key];
+              deletedKeys.push(key);
+            }
+          });
+        } else if (isMembersCollection) {
+          logger.error(
+            `Skipping user & contact creation for ${item} due to lack of email...`
+          );
+        }
+
+        context = { ...context, ...finalItem };
+        const validated = await this.importDataActions.validateItem(
+          finalItem,
+          importDef.attributeMappings,
+          context
+        );
+        if (!validated) {
+          console.error("Validation failed for item:", finalItem);
+          logger.error("Validation failed for item:", finalItem);
+          return;
+        }
 
-          context = { ...context, ...finalItem };
-          const validated = await this.importDataActions.validateItem(
+        if (
+          (importDef.type === "create" || !importDef.type) &&
+          !associatedDoc
+        ) {
+          const createdContext = await this.handleCreate(
+            context,
+            finalItem,
+            updateDefs,
+            createIdToUse
+          );
+          context = { ...context, ...createdContext };
+        } else {
+          const updatedContext = await this.handleUpdate(
+            context,
             finalItem,
+            importDef
+          );
+          context = { ...context, ...updatedContext };
+        }
+        const afterImportActionContext = structuredClone(context);
+        const attributeMappingsWithActions =
+          this.getAttributeMappingsWithActions(
             importDef.attributeMappings,
-            context
+            afterImportActionContext,
+            finalItem
           );
-          if (!validated) {
-            console.error("Validation failed for item:", finalItem);
-            logger.error("Validation failed for item:", finalItem);
-            return;
-          }
-
-          if (
-            (importDef.type === "create" || !importDef.type) &&
-            !associatedDoc
-          ) {
-            const createdContext = await this.handleCreate(
-              context,
-              finalItem,
-              updateDefs,
-              createIdToUse
-            );
-            context = { ...context, ...createdContext };
-          } else {
-            const updatedContext = await this.handleUpdate(
-              context,
-              finalItem,
-              importDef
-            );
-            context = { ...context, ...updatedContext };
-          }
-          const afterImportActionContext = structuredClone(context);
-          const attributeMappingsWithActions =
-            this.getAttributeMappingsWithActions(
-              importDef.attributeMappings,
-              afterImportActionContext,
-              finalItem
-            );
-          if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-            logger.info(
-              `Pushing to post-import actions queue for ${context.docId}`
-            );
-            const afterImportOperationContext = ContextObject.parse({
-              dbId: db.$id,
-              collectionId: collection.$id,
-              finalItem: finalItem,
-              attributeMappings: attributeMappingsWithActions,
-              context: afterImportActionContext,
-            });
-            await createOrFindAfterImportOperation(
-              this.database,
-              context.collId,
-              afterImportOperationContext
-            );
-            // this.postImportActionsQueue.push({
-            //   context: afterImportActionContext,
-            //   finalItem: finalItem,
-            //   attributeMappings: attributeMappingsWithActions,
-            // });
-          }
-        })
-      );
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          console.error("A process batch promise was rejected:", result.reason);
-          logger.error("An error occurred during creation: ", result.reason);
+        if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
+          logger.info(
+            `Pushing to post-import actions queue for ${context.docId}`
+          );
+          const afterImportOperationContext = ContextObject.parse({
+            dbId: db.$id,
+            collectionId: collection.$id,
+            finalItem: finalItem,
+            attributeMappings: attributeMappingsWithActions,
+            context: afterImportActionContext,
+          });
+          await createOrFindAfterImportOperation(
+            this.database,
+            context.collId,
+            afterImportOperationContext
+          );
+          // this.postImportActionsQueue.push({
+          //   context: afterImportActionContext,
+          //   finalItem: finalItem,
+          //   attributeMappings: attributeMappingsWithActions,
+          // });
         }
-      });
+      }
+      // const results = await Promise.allSettled(
+      //   batch.map(async (item: any) => {
+      //   })
+      // );
+      // results.forEach((result) => {
+      //   if (result.status === "rejected") {
+      //     console.error("A process batch promise was rejected:", result.reason);
+      //     logger.error("An error occurred during creation: ", result.reason);
+      //   }
+      // });
     }
   }
 

package/src/migrations/migrationHelper.ts

@@ -78,7 +78,6 @@ export const getAfterImportOperations = async (
     const query = [
       Query.equal("collectionId", collectionId),
       Query.equal("operationType", "afterImportAction"),
-      Query.equal("status", "ready"),
       Query.limit(100),
     ];
 
@@ -94,7 +93,7 @@ export const getAfterImportOperations = async (
     total = operations.total; // Update total with the latest fetch
     allOperations.push(...operations.documents);
 
-    if (operations.documents.length > 0) {
+    if (operations.documents.length > 0 && operations.documents.length >= 100) {
       lastDocumentId =
         operations.documents[operations.documents.length - 1].$id;
     }
@@ -134,7 +133,7 @@ export const setAllPendingAfterImportActionsToReady = async (
     }
 
     // Prepare for the next iteration in case there are more than 100 documents
-    if (operations.documents.length > 0) {
+    if (operations.documents.length > 0 && operations.documents.length >= 100) {
       lastDocumentId =
         operations.documents[operations.documents.length - 1].$id;
     } else {