appwrite-utils-cli 0.0.252 → 0.0.253

package/dist/main.js

@@ -13,6 +13,7 @@ async function main() {
     let generateSchemas = false;
     let importData = false;
     let wipeDocuments = false;
+    let shouldWriteFile = false;
     if (args.includes("--prod")) {
         runProd = true;
     }
@@ -40,6 +41,9 @@ async function main() {
     if (args.includes("--wipe-users") || args.includes("--wipeUsers")) {
         wipeUsers = true;
     }
+    if (args.includes("--write-data") || args.includes("--writeData")) {
+        shouldWriteFile = true;
+    }
     if (args.includes("--init")) {
         await controller.run({
             runProd: runProd,
@@ -53,6 +57,7 @@ async function main() {
             generateMockData: false,
             importData: false,
             checkDuplicates: false,
+            shouldWriteFile: shouldWriteFile,
         });
     }
     else {
@@ -68,6 +73,7 @@ async function main() {
             wipeUsers: wipeUsers,
             importData: importData,
             checkDuplicates: false,
+            shouldWriteFile: shouldWriteFile,
         });
     }
 }

package/dist/migrations/dataLoader.d.ts

@@ -1559,7 +1559,8 @@ export declare class DataLoader {
     private emailToUserIdMap;
     private phoneToUserIdMap;
     userExistsMap: Map<string, boolean>;
-    constructor(appwriteFolderPath: string, importDataActions: ImportDataActions, database: Databases, config: AppwriteConfig);
+    private shouldWriteFile;
+    constructor(appwriteFolderPath: string, importDataActions: ImportDataActions, database: Databases, config: AppwriteConfig, shouldWriteFile?: boolean);
     getCollectionKey(name: string): string;
     loadData(importDef: ImportDef): Promise<any[]>;
     checkMapValuesForId(newId: string, collectionName: string): string | false;
@@ -1578,7 +1579,7 @@ export declare class DataLoader {
     getAllUsers(): Promise<import("node-appwrite").Models.User<import("node-appwrite").Models.Preferences>[]>;
     start(dbId: string): Promise<void>;
     updateReferencesInRelatedCollections(): Promise<void>;
-    updateObjectWithNonNullishValues(sourceObject: Record<string, any>, inputObject: Record<string, any>): void;
+    private writeMapsToJsonFile;
     /**
      * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
      * and then returning the transformed item without user-specific keys.

package/dist/migrations/dataLoader.js

@@ -45,13 +45,15 @@ export class DataLoader {
     emailToUserIdMap = new Map();
     phoneToUserIdMap = new Map();
     userExistsMap = new Map();
+    shouldWriteFile = false;
     // Constructor to initialize the DataLoader with necessary configurations
-    constructor(appwriteFolderPath, importDataActions, database, config) {
+    constructor(appwriteFolderPath, importDataActions, database, config, shouldWriteFile) {
         this.appwriteFolderPath = appwriteFolderPath;
         this.importDataActions = importDataActions;
         this.database = database;
         this.usersController = new UsersController(config, database);
         this.config = config;
+        this.shouldWriteFile = shouldWriteFile || false;
     }
     // Helper method to generate a consistent key for collections
     getCollectionKey(name) {
@@ -156,7 +158,14 @@ export class DataLoader {
     async getAllUsers() {
         const users = new UsersController(this.config, this.database);
         const allUsers = await users.getAllUsers();
+        // Iterate over the users and setup our maps ahead of time for email and phone
         for (const user of allUsers) {
+            if (user.email) {
+                this.emailToUserIdMap.set(user.email, user.$id);
+            }
+            if (user.phone) {
+                this.phoneToUserIdMap.set(user.phone, user.$id);
+            }
             this.userExistsMap.set(user.$id, true);
         }
         return allUsers;
@@ -169,15 +178,6 @@ export class DataLoader {
         await this.setupMaps(dbId);
         const allUsers = await this.getAllUsers();
         console.log(`Fetched ${allUsers.length} users`);
-        // Iterate over the users and setup our maps ahead of time for email and phone
-        for (const user of allUsers) {
-            if (user.email) {
-                this.emailToUserIdMap.set(user.email, user.$id);
-            }
-            if (user.phone) {
-                this.phoneToUserIdMap.set(user.phone, user.$id);
-            }
-        }
         // Iterate over the configured databases to find the matching one
         for (const db of this.config.databases) {
             if (db.$id !== dbId) {
@@ -222,7 +222,9 @@ export class DataLoader {
         console.log("---------------------------------");
         console.log(`Data setup for database: ${dbId} completed`);
         console.log("---------------------------------");
-        // this.writeMapsToJsonFile();
+        if (this.shouldWriteFile) {
+            this.writeMapsToJsonFile();
+        }
     }
     async updateReferencesInRelatedCollections() {
         // Iterate over each collection configuration
@@ -299,24 +301,6 @@ export class DataLoader {
             }
         }
     }
-    updateObjectWithNonNullishValues(sourceObject, inputObject) {
-        // Iterate through the keys of the inputObject
-        for (const key of Object.keys(inputObject)) {
-            const inputValue = inputObject[key];
-            const sourceValue = sourceObject[key];
-            // Check if the inputObject's value for the current key is non-nullish
-            // and either the key doesn't exist in the sourceObject or its value is nullish
-            if (inputValue !== null &&
-                inputValue !== undefined &&
-                inputValue !== "" &&
-                (sourceValue === null ||
-                    sourceValue === undefined ||
-                    sourceValue === "")) {
-                // Update the sourceObject with the inputObject's value for the current key
-                sourceObject[key] = inputValue;
-            }
-        }
-    }
     // async updateReferencesInRelatedCollections() {
     //   // Process each collection defined in the config
     //   for (const collection of this.config.collections) {
@@ -368,43 +352,36 @@ export class DataLoader {
     //     }
     //   }
     // }
-    // private writeMapsToJsonFile() {
-    //   const outputDir = path.resolve(process.cwd());
-    //   const outputFile = path.join(outputDir, "dataLoaderOutput.json");
-    //   const dataToWrite = {
-    //     dataFromCollections: Array.from(this.importMap.entries()).map(
-    //       ([key, value]) => {
-    //         return {
-    //           collection: key,
-    //           data: value.data.map((item: any) => item.finalData),
-    //         };
-    //       }
-    //     ),
-    //     // Convert Maps to arrays of entries for serialization
-    //     mergedUserMap: Array.from(this.mergedUserMap.entries()),
-    //     // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
-    //     // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
-    //   };
-    //   // Use JSON.stringify with a replacer function to handle Maps
-    //   const replacer = (key: any, value: any) => {
-    //     if (value instanceof Map) {
-    //       return Array.from(value.entries());
-    //     }
-    //     return value;
-    //   };
-    //   fs.writeFile(
-    //     outputFile,
-    //     JSON.stringify(dataToWrite, replacer, 2),
-    //     "utf8",
-    //     (err) => {
-    //       if (err) {
-    //         console.error("Error writing data to JSON file:", err);
-    //         return;
-    //       }
-    //       console.log(`Data successfully written to ${outputFile}`);
-    //     }
-    //   );
-    // }
+    writeMapsToJsonFile() {
+        const outputDir = path.resolve(process.cwd());
+        const outputFile = path.join(outputDir, "dataLoaderOutput.json");
+        const dataToWrite = {
+            dataFromCollections: Array.from(this.importMap.entries()).map(([key, value]) => {
+                return {
+                    collection: key,
+                    data: value.data.map((item) => item.finalData),
+                };
+            }),
+            // Convert Maps to arrays of entries for serialization
+            mergedUserMap: Array.from(this.mergedUserMap.entries()),
+            // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
+            // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
+        };
+        // Use JSON.stringify with a replacer function to handle Maps
+        const replacer = (key, value) => {
+            if (value instanceof Map) {
+                return Array.from(value.entries());
+            }
+            return value;
+        };
+        fs.writeFile(outputFile, JSON.stringify(dataToWrite, replacer, 2), "utf8", (err) => {
+            if (err) {
+                console.error("Error writing data to JSON file:", err);
+                return;
+            }
+            console.log(`Data successfully written to ${outputFile}`);
+        });
+    }
     /**
      * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
      * and then returning the transformed item without user-specific keys.
@@ -499,6 +476,7 @@ export class DataLoader {
             this.oldIdToNewIdPerCollectionMap
                 .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
                 .get(this.getCollectionKey(collection.name));
+        console.log(`${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`);
         if (!operationId) {
             throw new Error(`No import operation found for collection ${collection.name}`);
         }
@@ -528,7 +506,6 @@ export class DataLoader {
                 // No existing user ID, generate a new unique ID
                 existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
                 transformedItem.userId = existingId; // Assign the new ID to the transformed data's userId field
-                transformedItem.docId = existingId;
             }
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, existingId);
@@ -542,7 +519,12 @@ export class DataLoader {
                     .get(this.getCollectionKey(collection.name))
                     ?.has(`${oldId}`)) {
                     // Found a duplicate oldId, now decide how to merge or handle these duplicates
-                    this.updateObjectWithNonNullishValues(currentData.data[0].finalData, transformedItem);
+                    for (const data of currentData.data) {
+                        if (data.finalData.docId === oldId ||
+                            data.finalData.userId === oldId) {
+                            Object.assign(data.finalData, transformedItem);
+                        }
+                    }
                 }
                 else {
                     // No duplicate found, simply map the oldId to the new itemId
@@ -556,7 +538,8 @@ export class DataLoader {
                 if ((currentUserData.data[i].finalData.docId === existingId ||
                     currentUserData.data[i].finalData.userId === existingId) &&
                     !_.isEqual(currentUserData.data[i], userData)) {
-                    this.updateObjectWithNonNullishValues(currentUserData.data[i].finalData, userData.finalData);
+                    Object.assign(currentUserData.data[i].finalData, transformedItem);
+                    Object.assign(currentUserData.data[i].rawData, item);
                     console.log("Merging user data", currentUserData.data[i].finalData);
                     this.importMap.set(this.getCollectionKey("users"), currentUserData);
                 }
@@ -572,7 +555,10 @@ export class DataLoader {
             for (let i = 0; i < currentData.data.length; i++) {
                 if (currentData.data[i].finalData.docId === existingId ||
                     currentData.data[i].finalData.userId === existingId) {
-                    this.updateObjectWithNonNullishValues(currentData.data[i].finalData, transformedItem);
+                    currentData.data[i].finalData = {
+                        ...currentData.data[i].finalData,
+                        ...transformedItem,
+                    };
                     currentData.data[i].importDef = newImportDef;
                     this.importMap.set(this.getCollectionKey(collection.name), currentData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
@@ -722,9 +708,8 @@ export class DataLoader {
                 }
             }
             else {
-                const currentCollectionData = this.importMap.get(this.getCollectionKey(collection.name));
-                const foundItem = currentCollectionData?.data.find((dataItem) => dataItem.rawData[importDef.primaryKeyField] === oldId);
-                newId = foundItem ? oldIdToNewIdMap?.get(`${oldId}`) : undefined;
+                logger.error(`No old ID found (to update another document with) in prepareUpdateData for ${collection.name}, ${JSON.stringify(item, null, 2)}`);
+                continue;
             }
             // Log an error and continue to the next item if no new ID is found
             if (!newId) {

package/dist/migrations/importController.js

@@ -58,7 +58,7 @@ export class ImportController {
             console.log(`Starting import data for database: ${db.name}`);
             console.log(`---------------------------------`);
             // await this.importCollections(db);
-            const dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config);
+            const dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
             await dataLoader.start(db.$id);
             await this.importCollections(db, dataLoader);
             await resolveAndUpdateRelationships(db.$id, this.database, this.config);

package/dist/utilsController.d.ts

@@ -13,6 +13,7 @@ export interface SetupOptions {
     generateMockData: boolean;
     importData: boolean;
     checkDuplicates: boolean;
+    shouldWriteFile: boolean;
 }
 export declare class UtilsController {
     private appwriteFolderPath;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "appwrite-utils-cli",
   "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
-  "version": "0.0.252",
+  "version": "0.0.253",
   "main": "src/main.ts",
   "type": "module",
   "repository": {

package/src/main.ts

@@ -16,6 +16,7 @@ async function main() {
   let generateSchemas = false;
   let importData = false;
   let wipeDocuments = false;
+  let shouldWriteFile = false;
   if (args.includes("--prod")) {
     runProd = true;
   }
@@ -43,6 +44,9 @@ async function main() {
   if (args.includes("--wipe-users") || args.includes("--wipeUsers")) {
     wipeUsers = true;
   }
+  if (args.includes("--write-data") || args.includes("--writeData")) {
+    shouldWriteFile = true;
+  }
   if (args.includes("--init")) {
     await controller.run({
       runProd: runProd,
@@ -56,6 +60,7 @@ async function main() {
       generateMockData: false,
       importData: false,
       checkDuplicates: false,
+      shouldWriteFile: shouldWriteFile,
     });
   } else {
     await controller.run({
@@ -70,6 +75,7 @@ async function main() {
       wipeUsers: wipeUsers,
       importData: importData,
       checkDuplicates: false,
+      shouldWriteFile: shouldWriteFile,
     });
   }
 }

package/src/migrations/dataLoader.ts

@@ -63,19 +63,22 @@ export class DataLoader {
   private emailToUserIdMap = new Map<string, string>();
   private phoneToUserIdMap = new Map<string, string>();
   userExistsMap = new Map<string, boolean>();
+  private shouldWriteFile = false;
 
   // Constructor to initialize the DataLoader with necessary configurations
   constructor(
     appwriteFolderPath: string,
     importDataActions: ImportDataActions,
     database: Databases,
-    config: AppwriteConfig
+    config: AppwriteConfig,
+    shouldWriteFile?: boolean
   ) {
     this.appwriteFolderPath = appwriteFolderPath;
     this.importDataActions = importDataActions;
     this.database = database;
     this.usersController = new UsersController(config, database);
     this.config = config;
+    this.shouldWriteFile = shouldWriteFile || false;
   }
 
   // Helper method to generate a consistent key for collections
@@ -213,7 +216,14 @@ export class DataLoader {
   async getAllUsers() {
     const users = new UsersController(this.config, this.database);
     const allUsers = await users.getAllUsers();
+    // Iterate over the users and setup our maps ahead of time for email and phone
     for (const user of allUsers) {
+      if (user.email) {
+        this.emailToUserIdMap.set(user.email, user.$id);
+      }
+      if (user.phone) {
+        this.phoneToUserIdMap.set(user.phone, user.$id);
+      }
       this.userExistsMap.set(user.$id, true);
     }
     return allUsers;
@@ -227,15 +237,6 @@ export class DataLoader {
     await this.setupMaps(dbId);
     const allUsers = await this.getAllUsers();
     console.log(`Fetched ${allUsers.length} users`);
-    // Iterate over the users and setup our maps ahead of time for email and phone
-    for (const user of allUsers) {
-      if (user.email) {
-        this.emailToUserIdMap.set(user.email, user.$id);
-      }
-      if (user.phone) {
-        this.phoneToUserIdMap.set(user.phone, user.$id);
-      }
-    }
     // Iterate over the configured databases to find the matching one
     for (const db of this.config.databases) {
       if (db.$id !== dbId) {
@@ -290,7 +291,9 @@ export class DataLoader {
     console.log("---------------------------------");
     console.log(`Data setup for database: ${dbId} completed`);
     console.log("---------------------------------");
-    // this.writeMapsToJsonFile();
+    if (this.shouldWriteFile) {
+      this.writeMapsToJsonFile();
+    }
   }
 
   async updateReferencesInRelatedCollections() {
@@ -389,31 +392,6 @@ export class DataLoader {
     }
   }
 
-  updateObjectWithNonNullishValues(
-    sourceObject: Record<string, any>,
-    inputObject: Record<string, any>
-  ): void {
-    // Iterate through the keys of the inputObject
-    for (const key of Object.keys(inputObject)) {
-      const inputValue = inputObject[key];
-      const sourceValue = sourceObject[key];
-
-      // Check if the inputObject's value for the current key is non-nullish
-      // and either the key doesn't exist in the sourceObject or its value is nullish
-      if (
-        inputValue !== null &&
-        inputValue !== undefined &&
-        inputValue !== "" &&
-        (sourceValue === null ||
-          sourceValue === undefined ||
-          sourceValue === "")
-      ) {
-        // Update the sourceObject with the inputObject's value for the current key
-        sourceObject[key] = inputValue;
-      }
-    }
-  }
-
   // async updateReferencesInRelatedCollections() {
   //   // Process each collection defined in the config
   //   for (const collection of this.config.collections) {
@@ -471,46 +449,46 @@ export class DataLoader {
   //   }
   // }
 
-  // private writeMapsToJsonFile() {
-  //   const outputDir = path.resolve(process.cwd());
-  //   const outputFile = path.join(outputDir, "dataLoaderOutput.json");
+  private writeMapsToJsonFile() {
+    const outputDir = path.resolve(process.cwd());
+    const outputFile = path.join(outputDir, "dataLoaderOutput.json");
 
-  //   const dataToWrite = {
-  //     dataFromCollections: Array.from(this.importMap.entries()).map(
-  //       ([key, value]) => {
-  //         return {
-  //           collection: key,
-  //           data: value.data.map((item: any) => item.finalData),
-  //         };
-  //       }
-  //     ),
-  //     // Convert Maps to arrays of entries for serialization
-  //     mergedUserMap: Array.from(this.mergedUserMap.entries()),
-  //     // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
-  //     // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
-  //   };
+    const dataToWrite = {
+      dataFromCollections: Array.from(this.importMap.entries()).map(
+        ([key, value]) => {
+          return {
+            collection: key,
+            data: value.data.map((item: any) => item.finalData),
+          };
+        }
+      ),
+      // Convert Maps to arrays of entries for serialization
+      mergedUserMap: Array.from(this.mergedUserMap.entries()),
+      // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
+      // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
+    };
 
-  //   // Use JSON.stringify with a replacer function to handle Maps
-  //   const replacer = (key: any, value: any) => {
-  //     if (value instanceof Map) {
-  //       return Array.from(value.entries());
-  //     }
-  //     return value;
-  //   };
+    // Use JSON.stringify with a replacer function to handle Maps
+    const replacer = (key: any, value: any) => {
+      if (value instanceof Map) {
+        return Array.from(value.entries());
+      }
+      return value;
+    };
 
-  //   fs.writeFile(
-  //     outputFile,
-  //     JSON.stringify(dataToWrite, replacer, 2),
-  //     "utf8",
-  //     (err) => {
-  //       if (err) {
-  //         console.error("Error writing data to JSON file:", err);
-  //         return;
-  //       }
-  //       console.log(`Data successfully written to ${outputFile}`);
-  //     }
-  //   );
-  // }
+    fs.writeFile(
+      outputFile,
+      JSON.stringify(dataToWrite, replacer, 2),
+      "utf8",
+      (err) => {
+        if (err) {
+          console.error("Error writing data to JSON file:", err);
+          return;
+        }
+        console.log(`Data successfully written to ${outputFile}`);
+      }
+    );
+  }
 
   /**
    * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
@@ -630,6 +608,9 @@ export class DataLoader {
       this.oldIdToNewIdPerCollectionMap
         .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
         .get(this.getCollectionKey(collection.name));
+    console.log(
+      `${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`
+    );
     if (!operationId) {
       throw new Error(
         `No import operation found for collection ${collection.name}`
@@ -679,7 +660,6 @@ export class DataLoader {
         // No existing user ID, generate a new unique ID
         existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
         transformedItem.userId = existingId; // Assign the new ID to the transformed data's userId field
-        transformedItem.docId = existingId;
       }
 
       // Create a context object for the item, including the new ID
@@ -699,10 +679,14 @@ export class DataLoader {
             ?.has(`${oldId}`)
         ) {
           // Found a duplicate oldId, now decide how to merge or handle these duplicates
-          this.updateObjectWithNonNullishValues(
-            currentData.data[0].finalData,
-            transformedItem
-          );
+          for (const data of currentData.data) {
+            if (
+              data.finalData.docId === oldId ||
+              data.finalData.userId === oldId
+            ) {
+              Object.assign(data.finalData, transformedItem);
+            }
+          }
         } else {
           // No duplicate found, simply map the oldId to the new itemId
           collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
@@ -718,10 +702,8 @@ export class DataLoader {
             currentUserData.data[i].finalData.userId === existingId) &&
           !_.isEqual(currentUserData.data[i], userData)
         ) {
-          this.updateObjectWithNonNullishValues(
-            currentUserData.data[i].finalData,
-            userData.finalData
-          );
+          Object.assign(currentUserData.data[i].finalData, transformedItem);
+          Object.assign(currentUserData.data[i].rawData, item);
           console.log("Merging user data", currentUserData.data[i].finalData);
           this.importMap.set(this.getCollectionKey("users"), currentUserData);
         }
@@ -744,10 +726,10 @@ export class DataLoader {
           currentData.data[i].finalData.docId === existingId ||
           currentData.data[i].finalData.userId === existingId
         ) {
-          this.updateObjectWithNonNullishValues(
-            currentData.data[i].finalData,
-            transformedItem
-          );
+          currentData.data[i].finalData = {
+            ...currentData.data[i].finalData,
+            ...transformedItem,
+          };
           currentData.data[i].importDef = newImportDef;
           this.importMap.set(
             this.getCollectionKey(collection.name),
@@ -962,13 +944,12 @@ export class DataLoader {
           }
         }
       } else {
-        const currentCollectionData = this.importMap.get(
-          this.getCollectionKey(collection.name)
-        );
-        const foundItem = currentCollectionData?.data.find(
-          (dataItem) => dataItem.rawData[importDef.primaryKeyField] === oldId
+        logger.error(
+          `No old ID found (to update another document with) in prepareUpdateData for ${
+            collection.name
+          }, ${JSON.stringify(item, null, 2)}`
         );
-        newId = foundItem ? oldIdToNewIdMap?.get(`${oldId}`) : undefined;
+        continue;
       }
       // Log an error and continue to the next item if no new ID is found
       if (!newId) {

package/src/migrations/importController.ts

@@ -107,7 +107,8 @@ export class ImportController {
         this.appwriteFolderPath,
         this.importDataActions,
         this.database,
-        this.config
+        this.config,
+        this.setupOptions.shouldWriteFile
       );
       await dataLoader.start(db.$id);
       await this.importCollections(db, dataLoader);

package/src/utilsController.ts

@@ -46,6 +46,7 @@ export interface SetupOptions {
   generateMockData: boolean;
   importData: boolean;
   checkDuplicates: boolean;
+  shouldWriteFile: boolean;
 }
 
 type CollectionConfig = AppwriteConfig["collections"];