appwrite-utils-cli 0.0.11 → 0.0.13

package/dist/migrations/importController.js

@@ -151,14 +151,13 @@ export class ImportController {
     async processBatch(db, collection, importDef, dataToImport, updateDefs = [], isMembersCollection = false) {
         for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
             const batch = dataToImport.slice(i, i + this.batchLimit);
-            batch.map(async (item) => {
+            const results = await Promise.allSettled(batch.map(async (item) => {
                 let context = this.createContext(db, collection, item);
                 let finalItem = await this.transformData(item, importDef.attributeMappings);
                 let createIdToUse = undefined;
                 let associatedDoc;
                 if (isMembersCollection &&
                     (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))) {
-                    console.log("Found members collection, creating user...");
                     const usersController = new UsersController(this.config, this.database);
                     const userToCreate = AuthUserCreateSchema.safeParse({
                         ...finalItem,
@@ -172,7 +171,6 @@ export class ImportController {
                     createIdToUse = user.$id;
                     context.docId = createIdToUse;
                     context = { ...context, ...user };
-                    console.log("Created user, deleting keys in finalItem that exist in user...");
                     const associatedDocFound = await this.database.listDocuments(db.$id, context.collId, [Query.equal("$id", createIdToUse)]);
                     if (associatedDocFound.documents.length > 0) {
                         associatedDoc = associatedDocFound.documents[0];
@@ -185,37 +183,28 @@ export class ImportController {
                             deletedKeys.push(key);
                         }
                     });
-                    console.log(`Set createIdToUse to ${createIdToUse}. Deleted keys: ${deletedKeys.join(", ")}.`);
                 }
                 else if (isMembersCollection) {
                     logger.error(`Skipping user & contact creation for ${item} due to lack of email...`);
                 }
                 context = { ...context, ...finalItem };
-                if (!(await this.importDataActions.validateItem(finalItem, importDef.attributeMappings, context))) {
+                const validated = await this.importDataActions.validateItem(finalItem, importDef.attributeMappings, context);
+                if (!validated) {
                     console.error("Validation failed for item:", finalItem);
+                    logger.error("Validation failed for item:", finalItem);
                     return;
                 }
-                let afterContext;
                 if ((importDef.type === "create" || !importDef.type) &&
                     !associatedDoc) {
                     const createdContext = await this.handleCreate(context, finalItem, updateDefs, createIdToUse);
-                    if (createdContext) {
-                        afterContext = createdContext;
-                    }
-                    logger.info(`Handled create for ${context.docId}}`);
+                    context = { ...context, ...createdContext };
                 }
                 else {
                     const updatedContext = await this.handleUpdate(context, finalItem, importDef);
-                    if (updatedContext) {
-                        afterContext = updatedContext;
-                    }
-                    logger.info(`Handled update for ${context.docId}`);
-                }
-                if (afterContext) {
-                    context = { ...context, ...afterContext };
+                    context = { ...context, ...updatedContext };
                 }
                 const afterImportActionContext = structuredClone(context);
-                const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, finalItem);
+                const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, afterImportActionContext, finalItem);
                 if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
                     logger.info(`Pushing to post-import actions queue for ${context.docId}`);
                     const afterImportOperationContext = ContextObject.parse({
@@ -232,21 +221,18 @@ export class ImportController {
                     //   attributeMappings: attributeMappingsWithActions,
                     // });
                 }
+            }));
+            results.forEach((result) => {
+                if (result.status === "rejected") {
+                    console.error("A process batch promise was rejected:", result.reason);
+                    logger.error("An error occurred during creation: ", result.reason);
+                }
             });
-            // results.forEach((result) => {
-            //   if (result.status === "rejected") {
-            //     console.error("A process batch promise was rejected:", result.reason);
-            //     logger.error("An error occurred during creation: ", result.reason);
-            //   }
-            // });
         }
     }
     async handleCreate(context, finalItem, updateDefs, id) {
         const existing = await documentExists(this.database, context.dbId, context.collId, finalItem);
         if (!existing) {
-            if (id) {
-                console.log(`Creating document with provided ID (member): ${id}`);
-            }
             const createdDoc = await this.database.createDocument(context.dbId, context.collId, id || ID.unique(), finalItem);
             context.docId = createdDoc.$id;
             context.createdDoc = createdDoc;
@@ -259,7 +245,6 @@ export class ImportController {
                     }
                 });
             }
-            console.log(`Created document ID: ${createdDoc.$id}`);
             return context;
         }
         else {
@@ -360,17 +345,18 @@ export class ImportController {
             for (const batch of resultsData) {
                 const actionOperation = ContextObject.parse(JSON.parse(batch.data));
                 const { context, finalItem, attributeMappings } = actionOperation;
+                if (finalItem.$id && !context.docId) {
+                    context.docId = finalItem.$id;
+                    logger.info(`Setting docId to ${finalItem.$id} because docId not found in context, batch ${batch.$id}, context is ${JSON.stringify(context)}`);
+                }
                 try {
                     await this.importDataActions.executeAfterImportActions(finalItem, attributeMappings, context);
                     // Mark batch as processed
                     await this.database.deleteDocument("migrations", "batches", batch.$id);
-                    await updateOperation(this.database, operation.$id, {
-                        status: "completed",
-                        batches: [],
-                    });
                 }
                 catch (error) {
-                    logger.error(`Failed to execute batch ${batch.$id}:`, error);
+                    logger.error(`Failed to execute batch ${batch.$id}:`, error, "Context is :", context);
+                    await this.database.deleteDocument("migrations", "batches", batch.$id);
                 }
             }
             // After processing all batches, update the operation status

package/dist/migrations/logging.js

@@ -1,7 +1,7 @@
 import winston from "winston";
 export const logger = winston.createLogger({
     level: "info",
-    format: winston.format.prettyPrint(),
+    format: winston.format.json({ space: 2 }),
     defaultMeta: { service: "appwrite-utils-cli" },
     transports: [
         //
@@ -9,6 +9,7 @@ export const logger = winston.createLogger({
         // - Write all logs with importance level of `info` or less to `combined.log`
         //
         new winston.transports.File({ filename: "error.log", level: "error" }),
+        new winston.transports.File({ filename: "warn.log", level: "warn" }),
         new winston.transports.File({ filename: "combined.log" }),
     ],
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -113,7 +113,6 @@ export class ImportController {
   async importCollections(db: ConfigDatabase) {
     const maxParallel = 3; // Maximum number of collections to process in parallel
     let activePromises: Promise<void>[] = []; // Array to keep track of active promises
-
     for (const collection of this.config.collections) {
       // Function that returns a promise for processing a single collection
       const processCollection = async (col: ConfigCollection) => {
@@ -201,7 +200,6 @@ export class ImportController {
       );
       await this.processBatch(db, collection, importDef, dataToImport);
     }
-
     await setAllPendingAfterImportActionsToReady(
       this.database,
       db.$id,
@@ -258,145 +256,127 @@ export class ImportController {
   ) {
     for (let i = 0; i < dataToImport.length; i += this.batchLimit) {
       const batch = dataToImport.slice(i, i + this.batchLimit);
-      batch.map(async (item: any) => {
-        let context = this.createContext(db, collection, item);
-        let finalItem = await this.transformData(
-          item,
-          importDef.attributeMappings
-        );
-        let createIdToUse: string | undefined = undefined;
-        let associatedDoc: Models.Document | undefined;
-        if (
-          isMembersCollection &&
-          (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
-        ) {
-          console.log("Found members collection, creating user...");
-          const usersController = new UsersController(
-            this.config,
-            this.database
-          );
-          const userToCreate = AuthUserCreateSchema.safeParse({
-            ...finalItem,
-          });
-          if (!userToCreate.success) {
-            console.error(userToCreate.error);
-            logger.error(userToCreate.error);
-            return;
-          }
-          const user = await usersController.createUserAndReturn(
-            userToCreate.data
+      const results = await Promise.allSettled(
+        batch.map(async (item: any) => {
+          let context = this.createContext(db, collection, item);
+          let finalItem = await this.transformData(
+            item,
+            importDef.attributeMappings
           );
-          createIdToUse = user.$id;
-          context.docId = createIdToUse;
-          context = { ...context, ...user };
-          console.log(
-            "Created user, deleting keys in finalItem that exist in user..."
-          );
-          const associatedDocFound = await this.database.listDocuments(
-            db.$id,
-            context.collId,
-            [Query.equal("$id", createIdToUse)]
-          );
-          if (associatedDocFound.documents.length > 0) {
-            associatedDoc = associatedDocFound.documents[0];
-          }
-          // Delete keys in finalItem that also exist in user
-          let deletedKeys: string[] = [];
-          Object.keys(finalItem).forEach((key) => {
-            if (user.hasOwnProperty(key)) {
-              delete finalItem[key];
-              deletedKeys.push(key);
+          let createIdToUse: string | undefined = undefined;
+          let associatedDoc: Models.Document | undefined;
+          if (
+            isMembersCollection &&
+            (finalItem.hasOwnProperty("email") || item.hasOwnProperty("phone"))
+          ) {
+            const usersController = new UsersController(
+              this.config,
+              this.database
+            );
+            const userToCreate = AuthUserCreateSchema.safeParse({
+              ...finalItem,
+            });
+            if (!userToCreate.success) {
+              console.error(userToCreate.error);
+              logger.error(userToCreate.error);
+              return;
             }
-          });
-          console.log(
-            `Set createIdToUse to ${createIdToUse}. Deleted keys: ${deletedKeys.join(
-              ", "
-            )}.`
-          );
-        } else if (isMembersCollection) {
-          logger.error(
-            `Skipping user & contact creation for ${item} due to lack of email...`
-          );
-        }
-
-        context = { ...context, ...finalItem };
+            const user = await usersController.createUserAndReturn(
+              userToCreate.data
+            );
+            createIdToUse = user.$id;
+            context.docId = createIdToUse;
+            context = { ...context, ...user };
+            const associatedDocFound = await this.database.listDocuments(
+              db.$id,
+              context.collId,
+              [Query.equal("$id", createIdToUse)]
+            );
+            if (associatedDocFound.documents.length > 0) {
+              associatedDoc = associatedDocFound.documents[0];
+            }
+            // Delete keys in finalItem that also exist in user
+            let deletedKeys: string[] = [];
+            Object.keys(finalItem).forEach((key) => {
+              if (user.hasOwnProperty(key)) {
+                delete finalItem[key];
+                deletedKeys.push(key);
+              }
+            });
+          } else if (isMembersCollection) {
+            logger.error(
+              `Skipping user & contact creation for ${item} due to lack of email...`
+            );
+          }
 
-        if (
-          !(await this.importDataActions.validateItem(
+          context = { ...context, ...finalItem };
+          const validated = await this.importDataActions.validateItem(
             finalItem,
             importDef.attributeMappings,
             context
-          ))
-        ) {
-          console.error("Validation failed for item:", finalItem);
-          return;
-        }
-
-        let afterContext;
-        if (
-          (importDef.type === "create" || !importDef.type) &&
-          !associatedDoc
-        ) {
-          const createdContext = await this.handleCreate(
-            context,
-            finalItem,
-            updateDefs,
-            createIdToUse
           );
-          if (createdContext) {
-            afterContext = createdContext;
+          if (!validated) {
+            console.error("Validation failed for item:", finalItem);
+            logger.error("Validation failed for item:", finalItem);
+            return;
           }
-          logger.info(`Handled create for ${context.docId}}`);
-        } else {
-          const updatedContext = await this.handleUpdate(
-            context,
-            finalItem,
-            importDef
-          );
-          if (updatedContext) {
-            afterContext = updatedContext;
+
+          if (
+            (importDef.type === "create" || !importDef.type) &&
+            !associatedDoc
+          ) {
+            const createdContext = await this.handleCreate(
+              context,
+              finalItem,
+              updateDefs,
+              createIdToUse
+            );
+            context = { ...context, ...createdContext };
+          } else {
+            const updatedContext = await this.handleUpdate(
+              context,
+              finalItem,
+              importDef
+            );
+            context = { ...context, ...updatedContext };
           }
-          logger.info(`Handled update for ${context.docId}`);
-        }
-        if (afterContext) {
-          context = { ...context, ...afterContext };
-        }
-        const afterImportActionContext = structuredClone(context);
-        const attributeMappingsWithActions =
-          this.getAttributeMappingsWithActions(
-            importDef.attributeMappings,
-            context,
-            finalItem
-          );
-        if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
-          logger.info(
-            `Pushing to post-import actions queue for ${context.docId}`
-          );
-          const afterImportOperationContext = ContextObject.parse({
-            dbId: db.$id,
-            collectionId: collection.$id,
-            finalItem: finalItem,
-            attributeMappings: attributeMappingsWithActions,
-            context: afterImportActionContext,
-          });
-          await createOrFindAfterImportOperation(
-            this.database,
-            context.collId,
-            afterImportOperationContext
-          );
-          // this.postImportActionsQueue.push({
-          //   context: afterImportActionContext,
-          //   finalItem: finalItem,
-          //   attributeMappings: attributeMappingsWithActions,
-          // });
+          const afterImportActionContext = structuredClone(context);
+          const attributeMappingsWithActions =
+            this.getAttributeMappingsWithActions(
+              importDef.attributeMappings,
+              afterImportActionContext,
+              finalItem
+            );
+          if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
+            logger.info(
+              `Pushing to post-import actions queue for ${context.docId}`
+            );
+            const afterImportOperationContext = ContextObject.parse({
+              dbId: db.$id,
+              collectionId: collection.$id,
+              finalItem: finalItem,
+              attributeMappings: attributeMappingsWithActions,
+              context: afterImportActionContext,
+            });
+            await createOrFindAfterImportOperation(
+              this.database,
+              context.collId,
+              afterImportOperationContext
+            );
+            // this.postImportActionsQueue.push({
+            //   context: afterImportActionContext,
+            //   finalItem: finalItem,
+            //   attributeMappings: attributeMappingsWithActions,
+            // });
+          }
+        })
+      );
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          console.error("A process batch promise was rejected:", result.reason);
+          logger.error("An error occurred during creation: ", result.reason);
         }
       });
-      // results.forEach((result) => {
-      //   if (result.status === "rejected") {
-      //     console.error("A process batch promise was rejected:", result.reason);
-      //     logger.error("An error occurred during creation: ", result.reason);
-      //   }
-      // });
     }
   }
 
@@ -413,9 +393,6 @@ export class ImportController {
       finalItem
     );
     if (!existing) {
-      if (id) {
-        console.log(`Creating document with provided ID (member): ${id}`);
-      }
       const createdDoc = await this.database.createDocument(
         context.dbId,
         context.collId,
@@ -437,8 +414,6 @@ export class ImportController {
           }
         });
       }
-
-      console.log(`Created document ID: ${createdDoc.$id}`);
       return context;
     } else {
       console.log("Document already exists, skipping creation.");
@@ -572,6 +547,16 @@ export class ImportController {
       for (const batch of resultsData) {
         const actionOperation = ContextObject.parse(JSON.parse(batch.data));
         const { context, finalItem, attributeMappings } = actionOperation;
+        if (finalItem.$id && !context.docId) {
+          context.docId = finalItem.$id;
+          logger.info(
+            `Setting docId to ${
+              finalItem.$id
+            } because docId not found in context, batch ${
+              batch.$id
+            }, context is ${JSON.stringify(context)}`
+          );
+        }
         try {
           await this.importDataActions.executeAfterImportActions(
             finalItem,
@@ -584,12 +569,18 @@ export class ImportController {
             "batches",
             batch.$id
           );
-          await updateOperation(this.database, operation.$id, {
-            status: "completed",
-            batches: [],
-          });
         } catch (error) {
-          logger.error(`Failed to execute batch ${batch.$id}:`, error);
+          logger.error(
+            `Failed to execute batch ${batch.$id}:`,
+            error,
+            "Context is :",
+            context
+          );
+          await this.database.deleteDocument(
+            "migrations",
+            "batches",
+            batch.$id
+          );
         }
       }
 

package/src/migrations/logging.ts

@@ -2,7 +2,7 @@ import winston from "winston";
 
 export const logger = winston.createLogger({
   level: "info",
-  format: winston.format.prettyPrint(),
+  format: winston.format.json({ space: 2 }),
   defaultMeta: { service: "appwrite-utils-cli" },
   transports: [
     //
@@ -10,6 +10,7 @@ export const logger = winston.createLogger({
     // - Write all logs with importance level of `info` or less to `combined.log`
     //
     new winston.transports.File({ filename: "error.log", level: "error" }),
+    new winston.transports.File({ filename: "warn.log", level: "warn" }),
     new winston.transports.File({ filename: "combined.log" }),
   ],
 });