appwrite-utils-cli 0.0.42 → 0.0.44

package/dist/migrations/converters.js

@@ -110,16 +110,10 @@ export const convertObjectByAttributeMappings = (obj, attributeMappings) => {
                     : value;
             }
             else {
-                result[mapping.targetKey] = null;
+                result[mapping.targetKey] = value ? value : null;
             }
         }
     }
-    // Ensure any keys in the original object that are not mapped are copied over
-    for (const key of Object.keys(obj)) {
-        if (!Object.keys(result).includes(key)) {
-            result[key] = obj[key];
-        }
-    }
     return result;
 };
 /**

package/dist/migrations/dataLoader.js

@@ -310,7 +310,7 @@ export class DataLoader {
                         continue;
                     }
                     // Prepare the update data for the collection
-                    await this.prepareUpdateData(db, collection, updateDef);
+                    this.prepareUpdateData(db, collection, updateDef);
                 }
             }
             console.log("Running update references");

package/dist/migrations/importController.d.ts

@@ -12,6 +12,7 @@ export declare class ImportController {
     private setupOptions;
     private documentCache;
     private batchLimit;
+    private hasImportedUsers;
     private postImportActionsQueue;
     constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions);
     run(): Promise<void>;

package/dist/migrations/importController.js

@@ -16,6 +16,7 @@ export class ImportController {
     setupOptions;
     documentCache;
     batchLimit = 25; // Define batch size limit
+    hasImportedUsers = false;
     postImportActionsQueue = [];
     constructor(config, database, storage, appwriteFolderPath, importDataActions, setupOptions) {
         this.config = config;
@@ -35,6 +36,7 @@ export class ImportController {
             (areCollectionNamesSame(db.name, this.config.databases[2].name) &&
                 this.setupOptions.runDev))
             .map((db) => db.name);
+        let dataLoader;
         for (let db of this.config.databases) {
             if (db.name.toLowerCase().trim().replace(" ", "") === "migrations" ||
                 !databasesToRun.includes(db.name)) {
@@ -52,8 +54,13 @@ export class ImportController {
             console.log(`Starting import data for database: ${db.name}`);
             console.log(`---------------------------------`);
             // await this.importCollections(db);
-            const dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
-            await dataLoader.start(db.$id);
+            if (!dataLoader) {
+                dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
+                await dataLoader.start(db.$id);
+            }
+            else {
+                console.log(`Using data from previous import run`);
+            }
             await this.importCollections(db, dataLoader);
             await resolveAndUpdateRelationships(db.$id, this.database, this.config);
             await this.executePostImportActions(db.$id, dataLoader);
@@ -81,7 +88,7 @@ export class ImportController {
                 }
                 return finalBatches;
             };
-            if (isUsersCollection) {
+            if (isUsersCollection && !this.hasImportedUsers) {
                 const usersDataMap = dataLoader.importMap.get(dataLoader.getCollectionKey("users"));
                 const usersData = usersDataMap?.data;
                 const usersController = new UsersController(this.config, this.database);
@@ -107,6 +114,10 @@ export class ImportController {
                                 !dataLoader.userExistsMap.has(itemId));
                         })
                             .map((item) => {
+                            dataLoader.userExistsMap.set(item.finalData.userId ||
+                                item.finalData.docId ||
+                                item.context.userId ||
+                                item.context.docId, true);
                             return usersController.createUserAndReturn(item.finalData);
                         });
                         const promiseResults = await Promise.allSettled(userBatchPromises);
@@ -120,6 +131,7 @@ export class ImportController {
                         }
                         console.log("Finished importing users batch");
                     }
+                    this.hasImportedUsers = true;
                     console.log("Finished importing users");
                 }
             }
@@ -142,23 +154,29 @@ export class ImportController {
                 const batches = dataSplit[i];
                 console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
                 const batchPromises = batches.map((item) => {
-                    const id = item.finalData.docId ||
-                        item.finalData.userId ||
-                        item.context.docId ||
-                        item.context.userId;
-                    if (item.finalData.hasOwnProperty("userId")) {
-                        delete item.finalData.userId;
-                    }
-                    if (item.finalData.hasOwnProperty("docId")) {
-                        delete item.finalData.docId;
+                    try {
+                        const id = item.finalData.docId ||
+                            item.finalData.userId ||
+                            item.context.docId ||
+                            item.context.userId;
+                        if (item.finalData.hasOwnProperty("userId")) {
+                            delete item.finalData.userId;
+                        }
+                        if (item.finalData.hasOwnProperty("docId")) {
+                            delete item.finalData.docId;
+                        }
+                        if (!item.finalData) {
+                            return Promise.resolve();
+                        }
+                        return tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, item.finalData));
                     }
-                    if (!item.finalData) {
+                    catch (error) {
+                        console.error(error);
                         return Promise.resolve();
                     }
-                    return tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, item.finalData));
                 });
                 // Wait for all promises in the current batch to resolve
-                await Promise.allSettled(batchPromises);
+                await Promise.all(batchPromises);
                 console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);
                 await updateOperation(this.database, importOperation.$id, {
                     progress: processedItems,

package/dist/migrations/users.js

@@ -45,7 +45,7 @@ export class UsersController {
     }
     async getAllUsers() {
         const allUsers = [];
-        const users = await this.users.list([Query.limit(200)]);
+        const users = await tryAwaitWithRetry(async () => await this.users.list([Query.limit(200)]));
         if (users.users.length === 0) {
             return [];
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.42",
+  "version": "0.0.44",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/migrations/converters.ts

@@ -123,18 +123,11 @@ export const convertObjectByAttributeMappings = (
           ? value.flat(Infinity)
           : value;
       } else {
-        result[mapping.targetKey] = null;
+        result[mapping.targetKey] = value ? value : null;
       }
     }
   }
 
-  // Ensure any keys in the original object that are not mapped are copied over
-  for (const key of Object.keys(obj)) {
-    if (!Object.keys(result).includes(key)) {
-      result[key] = obj[key];
-    }
-  }
-
   return result;
 };
 

package/src/migrations/dataLoader.ts

@@ -389,7 +389,7 @@ export class DataLoader {
             continue;
           }
           // Prepare the update data for the collection
-          await this.prepareUpdateData(db, collection, updateDef);
+          this.prepareUpdateData(db, collection, updateDef);
         }
       }
       console.log("Running update references");

package/src/migrations/importController.ts

@@ -36,6 +36,7 @@ export class ImportController {
   private setupOptions: SetupOptions;
   private documentCache: Map<string, any>;
   private batchLimit: number = 25; // Define batch size limit
+  private hasImportedUsers = false;
   private postImportActionsQueue: {
     context: any;
     finalItem: any;
@@ -71,7 +72,7 @@ export class ImportController {
             this.setupOptions.runDev)
       )
       .map((db) => db.name);
-
+    let dataLoader: DataLoader | undefined;
     for (let db of this.config.databases) {
       if (
         db.name.toLowerCase().trim().replace(" ", "") === "migrations" ||
@@ -91,14 +92,18 @@ export class ImportController {
       console.log(`Starting import data for database: ${db.name}`);
       console.log(`---------------------------------`);
       // await this.importCollections(db);
-      const dataLoader = new DataLoader(
-        this.appwriteFolderPath,
-        this.importDataActions,
-        this.database,
-        this.config,
-        this.setupOptions.shouldWriteFile
-      );
-      await dataLoader.start(db.$id);
+      if (!dataLoader) {
+        dataLoader = new DataLoader(
+          this.appwriteFolderPath,
+          this.importDataActions,
+          this.database,
+          this.config,
+          this.setupOptions.shouldWriteFile
+        );
+        await dataLoader.start(db.$id);
+      } else {
+        console.log(`Using data from previous import run`);
+      }
       await this.importCollections(db, dataLoader);
       await resolveAndUpdateRelationships(db.$id, this.database, this.config);
       await this.executePostImportActions(db.$id, dataLoader);
@@ -131,7 +136,7 @@ export class ImportController {
         return finalBatches;
       };
 
-      if (isUsersCollection) {
+      if (isUsersCollection && !this.hasImportedUsers) {
         const usersDataMap = dataLoader.importMap.get(
           dataLoader.getCollectionKey("users")
         );
@@ -160,6 +165,13 @@ export class ImportController {
                 );
               })
               .map((item) => {
+                dataLoader.userExistsMap.set(
+                  item.finalData.userId ||
+                    item.finalData.docId ||
+                    item.context.userId ||
+                    item.context.docId,
+                  true
+                );
                 return usersController.createUserAndReturn(item.finalData);
               });
             const promiseResults = await Promise.allSettled(userBatchPromises);
@@ -176,6 +188,7 @@ export class ImportController {
             }
             console.log("Finished importing users batch");
           }
+          this.hasImportedUsers = true;
           console.log("Finished importing users");
         }
       }
@@ -207,32 +220,37 @@ export class ImportController {
         const batches = dataSplit[i];
         console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
         const batchPromises = batches.map((item) => {
-          const id =
-            item.finalData.docId ||
-            item.finalData.userId ||
-            item.context.docId ||
-            item.context.userId;
-          if (item.finalData.hasOwnProperty("userId")) {
-            delete item.finalData.userId;
-          }
-          if (item.finalData.hasOwnProperty("docId")) {
-            delete item.finalData.docId;
-          }
-          if (!item.finalData) {
+          try {
+            const id =
+              item.finalData.docId ||
+              item.finalData.userId ||
+              item.context.docId ||
+              item.context.userId;
+            if (item.finalData.hasOwnProperty("userId")) {
+              delete item.finalData.userId;
+            }
+            if (item.finalData.hasOwnProperty("docId")) {
+              delete item.finalData.docId;
+            }
+            if (!item.finalData) {
+              return Promise.resolve();
+            }
+            return tryAwaitWithRetry(
+              async () =>
+                await this.database.createDocument(
+                  db.$id,
+                  collection.$id,
+                  id,
+                  item.finalData
+                )
+            );
+          } catch (error) {
+            console.error(error);
             return Promise.resolve();
           }
-          return tryAwaitWithRetry(
-            async () =>
-              await this.database.createDocument(
-                db.$id,
-                collection.$id,
-                id,
-                item.finalData
-              )
-          );
         });
         // Wait for all promises in the current batch to resolve
-        await Promise.allSettled(batchPromises);
+        await Promise.all(batchPromises);
         console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);
         await updateOperation(this.database, importOperation.$id, {
           progress: processedItems,

package/src/migrations/users.ts

@@ -68,7 +68,9 @@ export class UsersController {
 
   async getAllUsers() {
     const allUsers: Models.User<Models.Preferences>[] = [];
-    const users = await this.users.list([Query.limit(200)]);
+    const users = await tryAwaitWithRetry(
+      async () => await this.users.list([Query.limit(200)])
+    );
     if (users.users.length === 0) {
       return [];
     }