appwrite-utils-cli 0.0.17 → 0.0.19

package/dist/migrations/importController.d.ts

@@ -11,6 +11,7 @@ export declare class ImportController {
     private setupOptions;
     private documentCache;
     private batchLimit;
+    private postImportActionsQueue;
     constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions);
     run(): Promise<void>;
     importCollections(db: ConfigDatabase): Promise<void>;

package/dist/migrations/importController.js

@@ -21,11 +21,7 @@ export class ImportController {
     setupOptions;
     documentCache;
     batchLimit = 25; // Define batch size limit
-    // private postImportActionsQueue: {
-    //   context: any;
-    //   finalItem: any;
-    //   attributeMappings: AttributeMappings;
-    // }[] = [];
+    postImportActionsQueue = [];
     constructor(config, database, storage, appwriteFolderPath, importDataActions, setupOptions) {
         this.config = config;
         this.database = database;
@@ -211,19 +207,23 @@ export class ImportController {
                 const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, afterImportActionContext, finalItem);
                 if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
                     logger.info(`Pushing to post-import actions queue for ${context.docId}`);
-                    const afterImportOperationContext = ContextObject.parse({
-                        dbId: db.$id,
-                        collectionId: collection.$id,
-                        finalItem: finalItem,
-                        attributeMappings: attributeMappingsWithActions,
-                        context: afterImportActionContext,
-                    });
-                    await createOrFindAfterImportOperation(this.database, context.collId, afterImportOperationContext);
-                    // this.postImportActionsQueue.push({
-                    //   context: afterImportActionContext,
+                    // const afterImportOperationContext = ContextObject.parse({
+                    //   dbId: db.$id,
+                    //   collectionId: collection.$id,
                     //   finalItem: finalItem,
                     //   attributeMappings: attributeMappingsWithActions,
+                    //   context: afterImportActionContext,
                     // });
+                    // await createOrFindAfterImportOperation(
+                    //   this.database,
+                    //   context.collId,
+                    //   afterImportOperationContext
+                    // );
+                    this.postImportActionsQueue.push({
+                        context: afterImportActionContext,
+                        finalItem: finalItem,
+                        attributeMappings: attributeMappingsWithActions,
+                    });
                 }
             }));
             results.forEach((result) => {
@@ -318,16 +318,33 @@ export class ImportController {
         });
     }
     async executePostImportActions(dbId) {
-        const collectionActionsPromises = [];
-        for (const collection of this.config.collections) {
-            collectionActionsPromises.push(this.executeActionsInParallel(dbId, collection));
+        let actionQueue = [];
+        for (const action of this.postImportActionsQueue) {
+            actionQueue.push(this.importDataActions.executeAfterImportActions(action.finalItem, action.attributeMappings, action.context));
         }
-        const results = await Promise.allSettled(collectionActionsPromises);
-        results.forEach((result) => {
-            if (result.status === "rejected") {
-                console.error("A process batch promise was rejected:", result.reason);
-            }
-        });
+        const BATCH_LIMIT = 20;
+        const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
+        for (const queue of splitQueue) {
+            const results = await Promise.allSettled(queue);
+            results.forEach((result) => {
+                if (result.status === "rejected") {
+                    console.error("An action promise was rejected:", result.reason);
+                    logger.error(`An action promise was rejected: ${result.reason} -- ${JSON.stringify(result)}`);
+                }
+            });
+        }
+        // const collectionActionsPromises = [];
+        // for (const collection of this.config.collections) {
+        //   collectionActionsPromises.push(
+        //     this.executeActionsInParallel(dbId, collection)
+        //   );
+        // }
+        // const results = await Promise.allSettled(collectionActionsPromises);
+        // results.forEach((result) => {
+        //   if (result.status === "rejected") {
+        //     console.error("A process batch promise was rejected:", result.reason);
+        //   }
+        // });
     }
     async executeActionsInParallel(dbId, collection) {
         const collectionExists = await checkForCollection(this.database, dbId, collection);

package/dist/migrations/importDataActions.js

@@ -193,7 +193,7 @@ export class ImportDataActions {
                     resolvedString = resolvedString.replace(match[0], value);
                 }
                 else {
-                    logger.warn(`Failed to resolve ${template} in context: `, context);
+                    logger.warn(`Failed to resolve ${template} in context: `, JSON.stringify({ ...context, ...item }, null, 2));
                 }
             }
             // console.log(`Resolved string: ${resolvedString}`);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.17",
+  "version": "0.0.19",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/importController.ts

@@ -47,11 +47,11 @@ export class ImportController {
   private setupOptions: SetupOptions;
   private documentCache: Map<string, any>;
   private batchLimit: number = 25; // Define batch size limit
-  // private postImportActionsQueue: {
-  //   context: any;
-  //   finalItem: any;
-  //   attributeMappings: AttributeMappings;
-  // }[] = [];
+  private postImportActionsQueue: {
+    context: any;
+    finalItem: any;
+    attributeMappings: AttributeMappings;
+  }[] = [];
 
   constructor(
     config: AppwriteConfig,
@@ -357,23 +357,23 @@ export class ImportController {
             logger.info(
               `Pushing to post-import actions queue for ${context.docId}`
             );
-            const afterImportOperationContext = ContextObject.parse({
-              dbId: db.$id,
-              collectionId: collection.$id,
-              finalItem: finalItem,
-              attributeMappings: attributeMappingsWithActions,
-              context: afterImportActionContext,
-            });
-            await createOrFindAfterImportOperation(
-              this.database,
-              context.collId,
-              afterImportOperationContext
-            );
-            // this.postImportActionsQueue.push({
-            //   context: afterImportActionContext,
+            // const afterImportOperationContext = ContextObject.parse({
+            //   dbId: db.$id,
+            //   collectionId: collection.$id,
             //   finalItem: finalItem,
             //   attributeMappings: attributeMappingsWithActions,
+            //   context: afterImportActionContext,
             // });
+            // await createOrFindAfterImportOperation(
+            //   this.database,
+            //   context.collId,
+            //   afterImportOperationContext
+            // );
+            this.postImportActionsQueue.push({
+              context: afterImportActionContext,
+              finalItem: finalItem,
+              attributeMappings: attributeMappingsWithActions,
+            });
           }
         })
       );
@@ -512,18 +512,43 @@ export class ImportController {
   }
 
   async executePostImportActions(dbId: string) {
-    const collectionActionsPromises = [];
-    for (const collection of this.config.collections) {
-      collectionActionsPromises.push(
-        this.executeActionsInParallel(dbId, collection)
+    let actionQueue: Promise<any>[] = [];
+    for (const action of this.postImportActionsQueue) {
+      actionQueue.push(
+        this.importDataActions.executeAfterImportActions(
+          action.finalItem,
+          action.attributeMappings,
+          action.context
+        )
       );
     }
-    const results = await Promise.allSettled(collectionActionsPromises);
-    results.forEach((result) => {
-      if (result.status === "rejected") {
-        console.error("A process batch promise was rejected:", result.reason);
-      }
-    });
+    const BATCH_LIMIT = 20;
+    const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
+    for (const queue of splitQueue) {
+      const results = await Promise.allSettled(queue);
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          console.error("An action promise was rejected:", result.reason);
+          logger.error(
+            `An action promise was rejected: ${
+              result.reason
+            } -- ${JSON.stringify(result)}`
+          );
+        }
+      });
+    }
+    // const collectionActionsPromises = [];
+    // for (const collection of this.config.collections) {
+    //   collectionActionsPromises.push(
+    //     this.executeActionsInParallel(dbId, collection)
+    //   );
+    // }
+    // const results = await Promise.allSettled(collectionActionsPromises);
+    // results.forEach((result) => {
+    //   if (result.status === "rejected") {
+    //     console.error("A process batch promise was rejected:", result.reason);
+    //   }
+    // });
   }
 
   async executeActionsInParallel(dbId: string, collection: ConfigCollection) {

package/src/migrations/importDataActions.ts

@@ -285,7 +285,10 @@ export class ImportDataActions {
             : resolvedValue;
           resolvedString = resolvedString.replace(match[0], value);
         } else {
-          logger.warn(`Failed to resolve ${template} in context: `, context);
+          logger.warn(
+            `Failed to resolve ${template} in context: `,
+            JSON.stringify({ ...context, ...item }, null, 2)
+          );
         }
       }
       // console.log(`Resolved string: ${resolvedString}`);