appwrite-utils-cli 0.0.38 → 0.0.40

package/src/migrations/dataLoader.ts

@@ -103,32 +103,53 @@ export class DataLoader {
     // Create a new object to hold the merged result
     const result = { ...source };
 
-    Object.keys(update).forEach((key) => {
-      const sourceValue = source[key];
-      const updateValue = update[key];
-
-      // If the update value is an array, concatenate and remove duplicates
-      if (Array.isArray(updateValue)) {
-        const sourceArray = Array.isArray(sourceValue) ? sourceValue : [];
-        result[key] = [...new Set([...sourceArray, ...updateValue])];
-      }
-      // If the update value is an object, recursively merge
-      else if (
-        updateValue !== null &&
-        typeof updateValue === "object" &&
-        !(updateValue instanceof Date)
-      ) {
-        result[key] = this.mergeObjects(sourceValue, updateValue);
+    // Loop through the keys of the object we care about
+    for (const [key, value] of Object.entries(source)) {
+      // Check if the key exists in the target object
+      if (!Object.hasOwn(update, key)) {
+        // If the key doesn't exist, we can just skip it like bad cheese
+        continue;
       }
-      // If the update value is not nullish, overwrite the source value
-      else if (updateValue !== null && updateValue !== undefined) {
-        result[key] = updateValue;
+      if (update[key] === value) {
+        continue;
       }
-      // If the update value is nullish, keep the original value unless it doesn't exist
-      else if (sourceValue === undefined || sourceValue === null) {
-        result[key] = updateValue;
+      // If the value ain't here, we can just do whatever man
+      if (value === undefined || value === null || value === "") {
+        // If the update key is defined
+        if (
+          update[key] !== undefined &&
+          update[key] !== null &&
+          update[key] !== ""
+        ) {
+          // might as well use it eh?
+          result[key] = update[key];
+        }
+        // ELSE if the value is an array, because it would then not be === to those things above
+      } else if (Array.isArray(value)) {
+        // Get the update value
+        const updateValue = update[key];
+        // If the update value is an array, concatenate and remove duplicates
+        // and poopy data
+        if (Array.isArray(updateValue)) {
+          result[key] = [...new Set([...value, ...updateValue])].filter(
+            (item) => item !== null && item !== undefined && item !== ""
+          );
+        } else {
+          // If the update value is not an array, just use it
+          result[key] = [...value, updateValue].filter(
+            (item) => item !== null && item !== undefined && item !== ""
+          );
+        }
+      } else if (typeof value === "object") {
+        // If the value is an object, we need to merge it
+        if (typeof update[key] === "object") {
+          result[key] = this.mergeObjects(value, update[key]);
+        }
+      } else {
+        // Finally, the source value is defined, and not an array, so we don't care about the update value
+        continue;
       }
-    });
+    }
 
     return result;
   }
@@ -219,6 +240,9 @@ export class DataLoader {
       item,
       attributeMappings
     );
+    if (item["region"]) {
+      logger.info(`Converted item: ${JSON.stringify(convertedItem, null, 2)}`);
+    }
     // Run additional converter functions on the converted item, if any
     return this.importDataActions.runConverterFunctions(
       convertedItem,
@@ -346,11 +370,15 @@ export class DataLoader {
         for (const createDef of createDefs) {
           if (!isUsersCollection) {
             console.log(`${collection.name} is not users collection`);
-            this.prepareCreateData(db, collection, createDef);
+            await this.prepareCreateData(db, collection, createDef);
           } else {
             // Special handling for users collection if needed
             console.log(`${collection.name} is users collection`);
-            this.prepareUserCollectionCreateData(db, collection, createDef);
+            await this.prepareUserCollectionCreateData(
+              db,
+              collection,
+              createDef
+            );
           }
         }
         for (const updateDef of updateDefs) {
@@ -361,7 +389,7 @@ export class DataLoader {
             continue;
           }
           // Prepare the update data for the collection
-          this.prepareUpdateData(db, collection, updateDef);
+          await this.prepareUpdateData(db, collection, updateDef);
         }
       }
       console.log("Running update references");
@@ -531,7 +559,12 @@ export class DataLoader {
                   collectionData.data[i].context[idMapping.sourceField];
 
                 // Skip if value to match is missing or empty
-                if (!valueToMatch || _.isEmpty(valueToMatch)) continue;
+                if (
+                  !valueToMatch ||
+                  _.isEmpty(valueToMatch) ||
+                  valueToMatch === null
+                )
+                  continue;
 
                 const isFieldToSetArray = collectionConfig.attributes.find(
                   (attribute) => attribute.key === fieldToSetKey
@@ -814,11 +847,11 @@ export class DataLoader {
    * @param collection - The collection configuration.
    * @param importDef - The import definition containing the attribute mappings and other relevant info.
    */
-  prepareUserCollectionCreateData(
+  async prepareUserCollectionCreateData(
     db: ConfigDatabase,
     collection: CollectionCreate,
     importDef: ImportDef
-  ): void {
+  ): Promise<void> {
     // Load the raw data based on the import definition
     const rawData = this.loadData(importDef);
     const operationId = this.collectionImportOperations.get(
@@ -842,157 +875,136 @@ export class DataLoader {
         `No import operation found for collection ${collection.name}`
       );
     }
-    updateOperation(this.database, operationId, {
+    await updateOperation(this.database, operationId, {
       status: "ready",
       total: rawData.length,
-    }).then(() => {
-      // Retrieve the current user data and the current collection data from the import map
-      const currentUserData = this.importMap.get(
-        this.getCollectionKey("users")
+    });
+    // Retrieve the current user data and the current collection data from the import map
+    const currentUserData = this.importMap.get(this.getCollectionKey("users"));
+    const currentData = this.importMap.get(
+      this.getCollectionKey(collection.name)
+    );
+    // Log errors if the necessary data is not found in the import map
+    if (!currentUserData) {
+      logger.error(
+        `No data found for collection ${"users"} for createDef but it says it's supposed to have one...`
       );
-      const currentData = this.importMap.get(
-        this.getCollectionKey(collection.name)
+      return;
+    } else if (!currentData) {
+      logger.error(
+        `No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`
+      );
+      return;
+    }
+    // Iterate through each item in the raw data
+    for (const item of rawData) {
+      // Prepare user data, check for duplicates, and remove user-specific keys
+      let { transformedItem, existingId, userData } = this.prepareUserData(
+        item,
+        importDef.attributeMappings,
+        importDef.primaryKeyField,
+        this.getTrueUniqueId(this.getCollectionKey("users"))
       );
-      // Log errors if the necessary data is not found in the import map
-      if (!currentUserData) {
-        logger.error(
-          `No data found for collection ${"users"} for createDef but it says it's supposed to have one...`
-        );
-        return;
-      } else if (!currentData) {
-        logger.error(
-          `No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`
-        );
-        return;
-      }
-      // Iterate through each item in the raw data
-      for (const item of rawData) {
-        // Prepare user data, check for duplicates, and remove user-specific keys
-        let { transformedItem, existingId, userData } = this.prepareUserData(
-          item,
-          importDef.attributeMappings,
-          importDef.primaryKeyField,
-          this.getTrueUniqueId(this.getCollectionKey("users"))
-        );
 
-        logger.info(
-          `In create user -- transformedItem: ${JSON.stringify(
-            transformedItem,
-            null,
-            2
-          )}`
-        );
+      logger.info(
+        `In create user -- transformedItem: ${JSON.stringify(
+          transformedItem,
+          null,
+          2
+        )}`
+      );
 
-        // Generate a new unique ID for the item or use existing ID
-        if (!existingId) {
-          // No existing user ID, generate a new unique ID
-          existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
-          transformedItem.docId = existingId; // Assign the new ID to the transformed data's docId field
-        }
+      // Generate a new unique ID for the item or use existing ID
+      if (!existingId) {
+        // No existing user ID, generate a new unique ID
+        existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        transformedItem.docId = existingId; // Assign the new ID to the transformed data's docId field
+      }
 
-        // Create a context object for the item, including the new ID
-        let context = this.createContext(db, collection, item, existingId);
+      // Create a context object for the item, including the new ID
+      let context = this.createContext(db, collection, item, existingId);
 
-        // Merge the transformed data into the context
-        context = { ...context, ...transformedItem, ...userData.finalData };
+      // Merge the transformed data into the context
+      context = { ...context, ...transformedItem, ...userData.finalData };
 
-        // If a primary key field is defined, handle the ID mapping and check for duplicates
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
+      // If a primary key field is defined, handle the ID mapping and check for duplicates
+      if (importDef.primaryKeyField) {
+        const oldId = item[importDef.primaryKeyField];
 
-          // Check if the oldId already exists to handle potential duplicates
-          if (
-            this.oldIdToNewIdPerCollectionMap
-              .get(this.getCollectionKey(collection.name))
-              ?.has(`${oldId}`)
-          ) {
-            // Found a duplicate oldId, now decide how to merge or handle these duplicates
-            for (const data of currentData.data) {
-              if (
-                data.finalData.docId === oldId ||
-                data.finalData.userId === oldId
-              ) {
-                transformedItem = this.mergeObjects(
-                  data.finalData,
-                  transformedItem
-                );
-              }
+        // Check if the oldId already exists to handle potential duplicates
+        if (
+          this.oldIdToNewIdPerCollectionMap
+            .get(this.getCollectionKey(collection.name))
+            ?.has(`${oldId}`)
+        ) {
+          // Found a duplicate oldId, now decide how to merge or handle these duplicates
+          for (const data of currentData.data) {
+            if (
+              data.finalData.docId === oldId ||
+              data.finalData.userId === oldId
+            ) {
+              transformedItem = this.mergeObjects(
+                data.finalData,
+                transformedItem
+              );
             }
-          } else {
-            // No duplicate found, simply map the oldId to the new itemId
-            collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
           }
+        } else {
+          // No duplicate found, simply map the oldId to the new itemId
+          collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
         }
+      }
 
-        // Handle merging for currentUserData
-        for (let i = 0; i < currentUserData.data.length; i++) {
-          const currentUserDataItem = currentUserData.data[i];
-          const samePhones =
-            currentUserDataItem.finalData.phone &&
-            transformedItem.phone &&
-            currentUserDataItem.finalData.phone === transformedItem.phone;
-          const sameEmails =
-            currentUserDataItem.finalData.email &&
-            transformedItem.email &&
-            currentUserDataItem.finalData.email === transformedItem.email;
-          if (
-            (currentUserDataItem.finalData.docId === existingId ||
-              currentUserDataItem.finalData.userId === existingId) &&
-            (samePhones || sameEmails) &&
-            currentUserDataItem.finalData &&
+      // Handle merging for currentUserData
+      for (let i = 0; i < currentUserData.data.length; i++) {
+        const currentUserDataItem = currentUserData.data[i];
+        const samePhones =
+          currentUserDataItem.finalData.phone &&
+          transformedItem.phone &&
+          currentUserDataItem.finalData.phone === transformedItem.phone;
+        const sameEmails =
+          currentUserDataItem.finalData.email &&
+          transformedItem.email &&
+          currentUserDataItem.finalData.email === transformedItem.email;
+        if (
+          (currentUserDataItem.finalData.docId === existingId ||
+            currentUserDataItem.finalData.userId === existingId) &&
+          (samePhones || sameEmails) &&
+          currentUserDataItem.finalData &&
+          userData.finalData
+        ) {
+          const userDataMerged = this.mergeObjects(
+            currentUserData.data[i].finalData,
             userData.finalData
-          ) {
-            const userDataMerged = this.mergeObjects(
-              currentUserData.data[i].finalData,
-              userData.finalData
-            );
-            currentUserData.data[i].finalData = userDataMerged;
-            this.importMap.set(this.getCollectionKey("users"), currentUserData);
-          }
+          );
+          currentUserData.data[i].finalData = userDataMerged;
+          this.importMap.set(this.getCollectionKey("users"), currentUserData);
         }
-        // Update the attribute mappings with any actions that need to be performed post-import
-        const mappingsWithActions = this.getAttributeMappingsWithActions(
-          importDef.attributeMappings,
-          context,
-          transformedItem
-        );
-        // Update the import definition with the new attribute mappings
-        const newImportDef = {
-          ...importDef,
-          attributeMappings: mappingsWithActions,
-        };
+      }
+      // Update the attribute mappings with any actions that need to be performed post-import
+      const mappingsWithActions = this.getAttributeMappingsWithActions(
+        importDef.attributeMappings,
+        context,
+        transformedItem
+      );
+      // Update the import definition with the new attribute mappings
+      const newImportDef = {
+        ...importDef,
+        attributeMappings: mappingsWithActions,
+      };
 
-        let foundData = false;
-        for (let i = 0; i < currentData.data.length; i++) {
-          if (
-            currentData.data[i].finalData.docId === existingId ||
-            currentData.data[i].finalData.userId === existingId
-          ) {
-            currentData.data[i].finalData = this.mergeObjects(
-              currentData.data[i].finalData,
-              transformedItem
-            );
-            currentData.data[i].context = context;
-            currentData.data[i].importDef = newImportDef;
-            this.importMap.set(
-              this.getCollectionKey(collection.name),
-              currentData
-            );
-            this.oldIdToNewIdPerCollectionMap.set(
-              this.getCollectionKey(collection.name),
-              collectionOldIdToNewIdMap!
-            );
-            foundData = true;
-          }
-        }
-        if (!foundData) {
-          // Add new data to the associated collection
-          currentData.data.push({
-            rawData: item,
-            context: context,
-            importDef: newImportDef,
-            finalData: transformedItem,
-          });
+      let foundData = false;
+      for (let i = 0; i < currentData.data.length; i++) {
+        if (
+          currentData.data[i].finalData.docId === existingId ||
+          currentData.data[i].finalData.userId === existingId
+        ) {
+          currentData.data[i].finalData = this.mergeObjects(
+            currentData.data[i].finalData,
+            transformedItem
+          );
+          currentData.data[i].context = context;
+          currentData.data[i].importDef = newImportDef;
           this.importMap.set(
             this.getCollectionKey(collection.name),
             currentData
@@ -1001,9 +1013,24 @@ export class DataLoader {
             this.getCollectionKey(collection.name),
             collectionOldIdToNewIdMap!
           );
+          foundData = true;
         }
       }
-    });
+      if (!foundData) {
+        // Add new data to the associated collection
+        currentData.data.push({
+          rawData: item,
+          context: context,
+          importDef: newImportDef,
+          finalData: transformedItem,
+        });
+        this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        this.oldIdToNewIdPerCollectionMap.set(
+          this.getCollectionKey(collection.name),
+          collectionOldIdToNewIdMap!
+        );
+      }
+    }
   }
 
   /**
@@ -1014,11 +1041,11 @@ export class DataLoader {
    * @param collection - The collection configuration.
    * @param importDef - The import definition containing the attribute mappings and other relevant info.
    */
-  prepareCreateData(
+  async prepareCreateData(
     db: ConfigDatabase,
     collection: CollectionCreate,
     importDef: ImportDef
-  ): void {
+  ): Promise<void> {
     // Load the raw data based on the import definition
     const rawData = this.loadData(importDef);
     const operationId = this.collectionImportOperations.get(
@@ -1029,97 +1056,102 @@ export class DataLoader {
         `No import operation found for collection ${collection.name}`
       );
     }
-    updateOperation(this.database, operationId, {
+    await updateOperation(this.database, operationId, {
       status: "ready",
       total: rawData.length,
-    }).then(() => {
-      // Initialize a new map for old ID to new ID mappings
-      const oldIdToNewIdMapNew = new Map<string, string>();
-      // Retrieve or initialize the collection-specific old ID to new ID map
-      const collectionOldIdToNewIdMap =
-        this.oldIdToNewIdPerCollectionMap.get(
-          this.getCollectionKey(collection.name)
-        ) ||
-        this.oldIdToNewIdPerCollectionMap
-          .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
-          .get(this.getCollectionKey(collection.name));
-      console.log(
-        `${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`
+    });
+    // Initialize a new map for old ID to new ID mappings
+    const oldIdToNewIdMapNew = new Map<string, string>();
+    // Retrieve or initialize the collection-specific old ID to new ID map
+    const collectionOldIdToNewIdMap =
+      this.oldIdToNewIdPerCollectionMap.get(
+        this.getCollectionKey(collection.name)
+      ) ||
+      this.oldIdToNewIdPerCollectionMap
+        .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
+        .get(this.getCollectionKey(collection.name));
+    console.log(
+      `${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`
+    );
+    const isRegions = collection.name.toLowerCase() === "regions";
+    // Iterate through each item in the raw data
+    for (const item of rawData) {
+      // Generate a new unique ID for the item
+      const itemIdNew = this.getTrueUniqueId(
+        this.getCollectionKey(collection.name)
       );
-      // Iterate through each item in the raw data
-      for (const item of rawData) {
-        // Generate a new unique ID for the item
-        const itemIdNew = this.getTrueUniqueId(
-          this.getCollectionKey(collection.name)
-        );
-        // Retrieve the current collection data from the import map
-        const currentData = this.importMap.get(
-          this.getCollectionKey(collection.name)
-        );
-        // Create a context object for the item, including the new ID
-        let context = this.createContext(db, collection, item, itemIdNew);
-        // Transform the item data based on the attribute mappings
-        const transformedData = this.transformData(
-          item,
-          importDef.attributeMappings
-        );
-        // If a primary key field is defined, handle the ID mapping and check for duplicates
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
-          if (collectionOldIdToNewIdMap?.has(`${oldId}`)) {
-            logger.error(
-              `Collection ${collection.name} has multiple documents with the same primary key ${oldId}`
-            );
-            continue;
-          }
-          collectionOldIdToNewIdMap?.set(`${oldId}`, `${itemIdNew}`);
-        }
-        // Merge the transformed data into the context
-        context = { ...context, ...transformedData };
-        // Validate the item before proceeding
-        const isValid = this.importDataActions.validateItem(
-          transformedData,
-          importDef.attributeMappings,
-          context
-        );
-        if (!isValid) {
-          continue;
-        }
-        // Update the attribute mappings with any actions that need to be performed post-import
-        const mappingsWithActions = this.getAttributeMappingsWithActions(
-          importDef.attributeMappings,
-          context,
-          transformedData
+      if (isRegions) {
+        logger.info(`Creating region: ${JSON.stringify(item, null, 2)}`);
+      }
+      // Retrieve the current collection data from the import map
+      const currentData = this.importMap.get(
+        this.getCollectionKey(collection.name)
+      );
+      // Create a context object for the item, including the new ID
+      let context = this.createContext(db, collection, item, itemIdNew);
+      // Transform the item data based on the attribute mappings
+      const transformedData = this.transformData(
+        item,
+        importDef.attributeMappings
+      );
+      if (isRegions) {
+        logger.info(
+          `Transformed region: ${JSON.stringify(transformedData, null, 2)}`
         );
-        // Update the import definition with the new attribute mappings
-        const newImportDef = {
-          ...importDef,
-          attributeMappings: mappingsWithActions,
-        };
-        // If the current collection data exists, add the item with its context and final data
-        if (currentData && currentData.data) {
-          currentData.data.push({
-            rawData: item,
-            context: context,
-            importDef: newImportDef,
-            finalData: transformedData,
-          });
-          this.importMap.set(
-            this.getCollectionKey(collection.name),
-            currentData
-          );
-          this.oldIdToNewIdPerCollectionMap.set(
-            this.getCollectionKey(collection.name),
-            collectionOldIdToNewIdMap!
-          );
-        } else {
+      }
+      // If a primary key field is defined, handle the ID mapping and check for duplicates
+      if (importDef.primaryKeyField) {
+        const oldId = item[importDef.primaryKeyField];
+        if (collectionOldIdToNewIdMap?.has(`${oldId}`)) {
           logger.error(
-            `No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`
+            `Collection ${collection.name} has multiple documents with the same primary key ${oldId}`
           );
           continue;
         }
+        collectionOldIdToNewIdMap?.set(`${oldId}`, `${itemIdNew}`);
       }
-    });
+      // Merge the transformed data into the context
+      context = { ...context, ...transformedData };
+      // Validate the item before proceeding
+      const isValid = this.importDataActions.validateItem(
+        transformedData,
+        importDef.attributeMappings,
+        context
+      );
+      if (!isValid) {
+        continue;
+      }
+      // Update the attribute mappings with any actions that need to be performed post-import
+      const mappingsWithActions = this.getAttributeMappingsWithActions(
+        importDef.attributeMappings,
+        context,
+        transformedData
+      );
+      // Update the import definition with the new attribute mappings
+      const newImportDef = {
+        ...importDef,
+        attributeMappings: mappingsWithActions,
+      };
+      // If the current collection data exists, add the item with its context and final data
+      if (currentData && currentData.data) {
+        currentData.data.push({
+          rawData: item,
+          context: context,
+          importDef: newImportDef,
+          finalData: transformedData,
+        });
+        this.importMap.set(this.getCollectionKey(collection.name), currentData);
+        this.oldIdToNewIdPerCollectionMap.set(
+          this.getCollectionKey(collection.name),
+          collectionOldIdToNewIdMap!
+        );
+      } else {
+        logger.error(
+          `No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`
+        );
+        continue;
+      }
+    }
   }
 
   /**

package/src/migrations/importDataActions.ts

@@ -42,9 +42,19 @@ export class ImportDataActions {
     this.afterImportActionsDefinitions = afterImportActionsDefinitions;
   }
 
+  /**
+   * Runs converter functions on the item based on the provided attribute mappings.
+   *
+   * @param item - The item to be transformed.
+   * @param attributeMappings - The mappings that define how each attribute should be transformed.
+   * @returns The transformed item.
+   */
   runConverterFunctions(item: any, attributeMappings: AttributeMappings) {
     const conversionSchema = attributeMappings.reduce((schema, mapping) => {
       schema[mapping.targetKey] = (originalValue: any) => {
+        if (!mapping.converters) {
+          return originalValue;
+        }
         return mapping.converters?.reduce((value, converterName) => {
           let shouldProcessAsArray = false;
           if (
@@ -95,6 +105,7 @@ export class ImportDataActions {
   /**
    * Validates a single data item based on defined validation rules.
    * @param item The data item to validate.
+   * @param attributeMap The attribute mappings for the data item.
    * @param context The context for resolving templated parameters in validation rules.
    * @returns A promise that resolves to true if the item is valid, false otherwise.
    */