@njdamstra/appwrite-utils-cli 1.8.9

package/src/migrations/services/UserMappingService.ts

@@ -0,0 +1,345 @@
+import { z } from "zod";
+import { ID } from "node-appwrite";
+import type {
+  AttributeMappings,
+  AppwriteConfig,
+} from "@njdamstra/appwrite-utils";
+import { AuthUserCreateSchema } from "../../schemas/authUser.js";
+import { logger } from "../../shared/logging.js";
+import type { DataTransformationService } from "./DataTransformationService.js";
+
+/**
+ * Service responsible for user mapping and deduplication during import.
+ * Preserves all existing sophisticated user handling logic from DataLoader.
+ * Extracted to provide focused, testable user management operations.
+ */
+export class UserMappingService {
+  private config: AppwriteConfig;
+  private dataTransformationService: DataTransformationService;
+  
+  // Maps to hold email and phone to user ID mappings for uniqueness in User Accounts
+  private emailToUserIdMap = new Map<string, string>();
+  private phoneToUserIdMap = new Map<string, string>();
+  private userIdSet = new Set<string>();
+  
+  // Map to hold the merged user map for relationship resolution
+  // Will hold an array of the old user ID's that are mapped to the same new user ID
+  // For example, if there are two users with the same email, they will both be mapped to the same new user ID
+  // Prevents duplicate users with the other two maps below it and allows me to keep the old ID's
+  private mergedUserMap = new Map<string, string[]>();
+  
+  // Map to track existing users
+  public userExistsMap = new Map<string, boolean>();
+
+  constructor(
+    config: AppwriteConfig,
+    dataTransformationService: DataTransformationService
+  ) {
+    this.config = config;
+    this.dataTransformationService = dataTransformationService;
+  }
+
+  /**
+   * Initializes user maps with existing users from the system.
+   * Preserves existing user loading logic from DataLoader.
+   * 
+   * @param existingUsers - Array of existing users from the system
+   */
+  initializeWithExistingUsers(existingUsers: any[]): void {
+    // Clear existing maps
+    this.emailToUserIdMap.clear();
+    this.phoneToUserIdMap.clear();
+    this.userIdSet.clear();
+    this.userExistsMap.clear();
+
+    // Iterate over the users and setup our maps ahead of time for email and phone
+    for (const user of existingUsers) {
+      if (user.email) {
+        this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
+      }
+      if (user.phone) {
+        this.phoneToUserIdMap.set(user.phone, user.$id);
+      }
+      this.userExistsMap.set(user.$id, true);
+      this.userIdSet.add(user.$id);
+    }
+
+    logger.info(`Initialized user maps with ${existingUsers.length} existing users`);
+  }
+
+  /**
+   * Generates a unique ID that doesn't conflict with existing user IDs.
+   * Preserves existing unique ID generation logic from DataLoader.
+   * 
+   * @param collectionName - The collection name for context
+   * @returns A truly unique ID
+   */
+  getTrueUniqueUserId(collectionName: string): string {
+    let newId = ID.unique();
+    let condition =
+      this.userExistsMap.has(newId) ||
+      this.userIdSet.has(newId) ||
+      Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+      Array.from(this.phoneToUserIdMap.values()).includes(newId);
+      
+    while (condition) {
+      newId = ID.unique();
+      condition =
+        this.userExistsMap.has(newId) ||
+        this.userIdSet.has(newId) ||
+        Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+        Array.from(this.phoneToUserIdMap.values()).includes(newId);
+    }
+    return newId;
+  }
+
+  /**
+   * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
+   * and then returning the transformed item without user-specific keys.
+   * 
+   * Preserves existing sophisticated user deduplication logic from DataLoader.
+   *
+   * @param item - The raw item to be processed.
+   * @param attributeMappings - The attribute mappings for the item.
+   * @param primaryKeyField - The primary key field name
+   * @param newId - The proposed new ID for the user
+   * @returns Object containing transformed item, existing ID if duplicate, and user data
+   */
+  prepareUserData(
+    item: any,
+    attributeMappings: AttributeMappings,
+    primaryKeyField: string,
+    newId: string
+  ): {
+    transformedItem: any;
+    existingId: string | undefined;
+    userData: {
+      rawData: any;
+      finalData: z.infer<typeof AuthUserCreateSchema>;
+    };
+  } {
+    // Ensure we have a truly unique ID
+    if (
+      this.userIdSet.has(newId) ||
+      this.userExistsMap.has(newId) ||
+      Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+      Array.from(this.phoneToUserIdMap.values()).includes(newId)
+    ) {
+      newId = this.getTrueUniqueUserId("users");
+    }
+
+    let transformedItem = this.dataTransformationService.transformData(item, attributeMappings);
+    let userData = AuthUserCreateSchema.safeParse(transformedItem);
+    
+    if (userData.data?.email) {
+      userData.data.email = userData.data.email.toLowerCase();
+    }
+    
+    if (!userData.success || !(userData.data.email || userData.data.phone)) {
+      logger.error(
+        `Invalid user data: ${JSON.stringify(
+          userData.error?.issues,
+          undefined,
+          2
+        )} or missing email/phone`
+      );
+
+      const userKeys = ["email", "phone", "name", "labels", "prefs"];
+      userKeys.forEach((key) => {
+        if (transformedItem.hasOwnProperty(key)) {
+          delete transformedItem[key];
+        }
+      });
+      
+      return {
+        transformedItem,
+        existingId: undefined,
+        userData: {
+          rawData: item,
+          finalData: transformedItem,
+        },
+      };
+    }
+
+    const email = userData.data.email?.toLowerCase();
+    const phone = userData.data.phone;
+    let existingId: string | undefined;
+
+    // Check for duplicate email and phone
+    if (email && this.emailToUserIdMap.has(email)) {
+      existingId = this.emailToUserIdMap.get(email);
+      if (phone && !this.phoneToUserIdMap.has(phone)) {
+        this.phoneToUserIdMap.set(phone, newId);
+      }
+    } else if (phone && this.phoneToUserIdMap.has(phone)) {
+      existingId = this.phoneToUserIdMap.get(phone);
+      if (email && !this.emailToUserIdMap.has(email)) {
+        this.emailToUserIdMap.set(email, newId);
+      }
+    } else {
+      if (email) this.emailToUserIdMap.set(email, newId);
+      if (phone) this.phoneToUserIdMap.set(phone, newId);
+    }
+
+    if (existingId) {
+      userData.data.userId = existingId;
+      const mergedUsers = this.mergedUserMap.get(existingId) || [];
+      mergedUsers.push(`${item[primaryKeyField]}`);
+      this.mergedUserMap.set(existingId, mergedUsers);
+
+      const userKeys = ["email", "phone", "name", "labels", "prefs"];
+      userKeys.forEach((key) => {
+        if (transformedItem.hasOwnProperty(key)) {
+          delete transformedItem[key];
+        }
+      });
+      
+      return {
+        transformedItem: {
+          ...transformedItem,
+          userId: existingId,
+          docId: existingId,
+        },
+        existingId,
+        userData: {
+          rawData: item,
+          finalData: userData.data,
+        },
+      };
+    } else {
+      existingId = newId;
+      userData.data.userId = existingId;
+    }
+
+    const userKeys = ["email", "phone", "name", "labels", "prefs"];
+    userKeys.forEach((key) => {
+      if (transformedItem.hasOwnProperty(key)) {
+        delete transformedItem[key];
+      }
+    });
+
+    this.userIdSet.add(existingId);
+
+    return {
+      transformedItem: {
+        ...transformedItem,
+        userId: existingId,
+        docId: existingId,
+      },
+      existingId,
+      userData: {
+        rawData: item,
+        finalData: userData.data,
+      },
+    };
+  }
+
+  /**
+   * Checks if a collection is the users collection based on configuration.
+   * 
+   * @param collectionName - The collection name to check
+   * @returns True if this is the users collection
+   */
+  isUsersCollection(collectionName: string): boolean {
+    return !!(
+      this.config.usersCollectionName &&
+      this.getCollectionKey(this.config.usersCollectionName) ===
+        this.getCollectionKey(collectionName)
+    );
+  }
+
+  /**
+   * Helper method to generate a consistent key for collections.
+   * Preserves existing logic from DataLoader.
+   */
+  private getCollectionKey(name: string): string {
+    return name.toLowerCase().replace(" ", "");
+  }
+
+  /**
+   * Gets merged user mappings for relationship resolution.
+   * 
+   * @returns Map of merged user IDs to arrays of original IDs
+   */
+  getMergedUserMap(): Map<string, string[]> {
+    return new Map(this.mergedUserMap);
+  }
+
+  /**
+   * Finds the new user ID for an old user ID, considering merged users.
+   * Preserves existing merged user lookup logic from DataLoader.
+   * 
+   * @param oldId - The old user ID to look up
+   * @returns The new user ID if found, otherwise the original ID
+   */
+  findNewUserIdForOldId(oldId: string): string {
+    // Check if this old ID was merged into another user
+    for (const [newUserId, oldUserIds] of this.mergedUserMap.entries()) {
+      if (oldUserIds.includes(`${oldId}`)) {
+        return newUserId;
+      }
+    }
+    
+    // If not found in merged users, return the original ID
+    return oldId;
+  }
+
+  /**
+   * Merges user data from duplicate entries.
+   * Preserves existing user data merging logic.
+   * 
+   * @param existingUserData - The existing user data
+   * @param newUserData - The new user data to merge
+   * @returns Merged user data
+   */
+  mergeUserData(existingUserData: any, newUserData: any): any {
+    return this.dataTransformationService.mergeObjects(existingUserData, newUserData);
+  }
+
+  /**
+   * Gets statistics about user mapping operations.
+   * Useful for import planning and reporting.
+   * 
+   * @returns User mapping statistics
+   */
+  getStatistics(): {
+    totalExistingUsers: number;
+    emailMappings: number;
+    phoneMappings: number;
+    mergedUsers: number;
+    totalMergedOldIds: number;
+  } {
+    let totalMergedOldIds = 0;
+    for (const oldIds of this.mergedUserMap.values()) {
+      totalMergedOldIds += oldIds.length;
+    }
+
+    return {
+      totalExistingUsers: this.userExistsMap.size,
+      emailMappings: this.emailToUserIdMap.size,
+      phoneMappings: this.phoneToUserIdMap.size,
+      mergedUsers: this.mergedUserMap.size,
+      totalMergedOldIds,
+    };
+  }
+
+  /**
+   * Validates user mapping configuration.
+   * Ensures the user mapping setup is correct before import.
+   * 
+   * @returns Array of validation errors (empty if valid)
+   */
+  validateConfiguration(): string[] {
+    const errors: string[] = [];
+
+    if (!this.config.usersCollectionName) {
+      errors.push("usersCollectionName is not configured");
+    }
+
+    if (!this.config.documentBucketId) {
+      errors.push("documentBucketId is not configured");
+    }
+
+    return errors;
+  }
+}

package/src/migrations/services/ValidationService.ts

@@ -0,0 +1,349 @@
+import type {
+  AttributeMappings,
+  ImportDef,
+  CollectionCreate,
+} from "@njdamstra/appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+import { logger } from "../../shared/logging.js";
+import fs from "fs";
+import path from "path";
+
+/**
+ * Service responsible for validation during import operations.
+ * Provides centralized validation logic extracted from ImportDataActions and DataLoader.
+ */
+export class ValidationService {
+  private importDataActions: ImportDataActions;
+
+  constructor(importDataActions: ImportDataActions) {
+    this.importDataActions = importDataActions;
+  }
+
+  /**
+   * Validates a single data item based on defined validation rules.
+   * Preserves existing validation logic from ImportDataActions.
+   * 
+   * @param item - The data item to validate
+   * @param attributeMappings - The attribute mappings for the data item
+   * @param context - The context for resolving templated parameters in validation rules
+   * @returns True if the item is valid, false otherwise
+   */
+  validateDataItem(
+    item: any,
+    attributeMappings: AttributeMappings,
+    context: { [key: string]: any }
+  ): boolean {
+    try {
+      return this.importDataActions.validateItem(item, attributeMappings, context);
+    } catch (error) {
+      logger.error(`Validation error for item: ${JSON.stringify(item, null, 2)}`, error);
+      return false;
+    }
+  }
+
+  /**
+   * Validates an import definition before processing begins.
+   * Provides early validation to catch configuration issues.
+   * 
+   * @param importDef - The import definition to validate
+   * @returns Array of validation errors (empty if valid)
+   */
+  validateImportDefinition(importDef: ImportDef): string[] {
+    const errors: string[] = [];
+
+    // Validate required fields
+    if (!importDef.filePath) {
+      errors.push("filePath is required");
+    }
+
+    if (!importDef.attributeMappings || importDef.attributeMappings.length === 0) {
+      errors.push("attributeMappings are required");
+    }
+
+    // Validate attribute mappings
+    if (importDef.attributeMappings) {
+      for (let i = 0; i < importDef.attributeMappings.length; i++) {
+        const mapping = importDef.attributeMappings[i];
+        
+        if (!mapping.targetKey) {
+          errors.push(`attributeMappings[${i}]: targetKey is required`);
+        }
+
+        if (!mapping.oldKey && !mapping.oldKeys && mapping.valueToSet === undefined) {
+          errors.push(
+            `attributeMappings[${i}]: must have either oldKey, oldKeys, or valueToSet`
+          );
+        }
+
+        // Validate file data if present
+        if (mapping.fileData) {
+          if (!mapping.fileData.path) {
+            errors.push(`attributeMappings[${i}].fileData: path is required`);
+          }
+          if (!mapping.fileData.name) {
+            errors.push(`attributeMappings[${i}].fileData: name is required`);
+          }
+        }
+
+        // Validate converters if present
+        if (mapping.converters && !Array.isArray(mapping.converters)) {
+          errors.push(`attributeMappings[${i}].converters: must be an array`);
+        }
+
+        // Validate validation actions if present
+        if (mapping.validationActions) {
+          if (!Array.isArray(mapping.validationActions)) {
+            errors.push(`attributeMappings[${i}].validationActions: must be an array`);
+          } else {
+            for (let j = 0; j < mapping.validationActions.length; j++) {
+              const action = mapping.validationActions[j];
+              if (!action.action) {
+                errors.push(
+                  `attributeMappings[${i}].validationActions[${j}]: action is required`
+                );
+              }
+              if (!Array.isArray(action.params)) {
+                errors.push(
+                  `attributeMappings[${i}].validationActions[${j}]: params must be an array`
+                );
+              }
+            }
+          }
+        }
+
+        // Validate post-import actions if present
+        if (mapping.postImportActions) {
+          if (!Array.isArray(mapping.postImportActions)) {
+            errors.push(`attributeMappings[${i}].postImportActions: must be an array`);
+          } else {
+            for (let j = 0; j < mapping.postImportActions.length; j++) {
+              const action = mapping.postImportActions[j];
+              if (!action.action) {
+                errors.push(
+                  `attributeMappings[${i}].postImportActions[${j}]: action is required`
+                );
+              }
+              if (!Array.isArray(action.params)) {
+                errors.push(
+                  `attributeMappings[${i}].postImportActions[${j}]: params must be an array`
+                );
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // Validate ID mappings if present
+    if (importDef.idMappings) {
+      for (let i = 0; i < importDef.idMappings.length; i++) {
+        const idMapping = importDef.idMappings[i];
+        
+        if (!idMapping.sourceField) {
+          errors.push(`idMappings[${i}]: sourceField is required`);
+        }
+        if (!idMapping.targetField) {
+          errors.push(`idMappings[${i}]: targetField is required`);
+        }
+        if (!idMapping.targetCollection) {
+          errors.push(`idMappings[${i}]: targetCollection is required`);
+        }
+      }
+    }
+
+    // Validate update mapping if present
+    if (importDef.updateMapping) {
+      if (!importDef.updateMapping.originalIdField) {
+        errors.push("updateMapping: originalIdField is required");
+      }
+      if (!importDef.updateMapping.targetField) {
+        errors.push("updateMapping: targetField is required");
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validates a collection configuration for import compatibility.
+   * Ensures the collection has the necessary attributes for import operations.
+   * 
+   * @param collection - The collection to validate
+   * @param importDefs - The import definitions that will be used with this collection
+   * @returns Array of validation errors (empty if valid)
+   */
+  validateCollectionForImport(
+    collection: CollectionCreate,
+    importDefs: ImportDef[]
+  ): string[] {
+    const errors: string[] = [];
+
+    if (!collection.name) {
+      errors.push("Collection name is required");
+    }
+
+    if (!collection.attributes || collection.attributes.length === 0) {
+      errors.push("Collection must have attributes defined");
+    }
+
+    // Validate that target keys in import definitions exist as attributes
+    for (const importDef of importDefs) {
+      for (const mapping of importDef.attributeMappings) {
+        const attributeExists = collection.attributes?.some(
+          attr => attr.key === mapping.targetKey
+        );
+        
+        if (!attributeExists) {
+          errors.push(
+            `Attribute '${mapping.targetKey}' referenced in import mapping not found in collection`
+          );
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validates data consistency across multiple collections.
+   * Checks for relationship integrity and data consistency.
+   * 
+   * @param collections - Array of collections to validate
+   * @returns Array of validation errors (empty if valid)
+   */
+  validateCrossCollectionConsistency(
+    collections: CollectionCreate[]
+  ): string[] {
+    const errors: string[] = [];
+    const collectionNames = new Set(collections.map(c => c.name));
+
+    for (const collection of collections) {
+      if (!collection.importDefs) continue;
+
+      for (const importDef of collection.importDefs) {
+        if (!importDef.idMappings) continue;
+
+        for (const idMapping of importDef.idMappings) {
+          // Check if target collection exists
+          if (!collectionNames.has(idMapping.targetCollection)) {
+            errors.push(
+              `Collection '${collection.name}' references non-existent target collection '${idMapping.targetCollection}' in ID mapping`
+            );
+          }
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Performs comprehensive pre-import validation.
+   * Validates all aspects of the import configuration before starting.
+   * 
+   * @param collections - Collections that will be imported
+   * @param appwriteFolderPath - Path to the appwrite folder for file validation
+   * @returns Validation result with errors and warnings
+   */
+  performPreImportValidation(
+    collections: CollectionCreate[],
+    appwriteFolderPath: string
+  ): {
+    errors: string[];
+    warnings: string[];
+    isValid: boolean;
+  } {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+
+    // Validate each collection
+    for (const collection of collections) {
+      const collectionErrors = this.validateCollectionForImport(
+        collection,
+        collection.importDefs || []
+      );
+      errors.push(...collectionErrors.map(err => `${collection.name}: ${err}`));
+
+      // Validate each import definition
+      if (collection.importDefs) {
+        for (let i = 0; i < collection.importDefs.length; i++) {
+          const importDef = collection.importDefs[i];
+          const importDefErrors = this.validateImportDefinition(importDef);
+          errors.push(
+            ...importDefErrors.map(
+              err => `${collection.name}.importDefs[${i}]: ${err}`
+            )
+          );
+
+          // Check if import file exists
+          try {
+            const filePath = path.resolve(appwriteFolderPath, importDef.filePath);
+            if (!fs.existsSync(filePath)) {
+              errors.push(
+                `${collection.name}.importDefs[${i}]: Import file not found: ${filePath}`
+              );
+            }
+          } catch (error) {
+            warnings.push(
+              `${collection.name}.importDefs[${i}]: Could not validate file existence: ${error}`
+            );
+          }
+        }
+      }
+    }
+
+    // Validate cross-collection consistency
+    const crossCollectionErrors = this.validateCrossCollectionConsistency(collections);
+    errors.push(...crossCollectionErrors);
+
+    return {
+      errors,
+      warnings,
+      isValid: errors.length === 0,
+    };
+  }
+
+  /**
+   * Validates import progress and data integrity during import.
+   * Can be used for periodic validation during long-running imports.
+   * 
+   * @param processedItems - Number of items processed so far
+   * @param totalItems - Total number of items to process
+   * @param errorCount - Number of errors encountered
+   * @returns Validation status with recommendations
+   */
+  validateImportProgress(
+    processedItems: number,
+    totalItems: number,
+    errorCount: number
+  ): {
+    status: "healthy" | "warning" | "critical";
+    message: string;
+    shouldContinue: boolean;
+  } {
+    const errorRate = errorCount / Math.max(processedItems, 1);
+    const progress = processedItems / totalItems;
+
+    if (errorRate > 0.5) {
+      return {
+        status: "critical",
+        message: `Error rate too high: ${(errorRate * 100).toFixed(1)}%. Consider stopping import.`,
+        shouldContinue: false,
+      };
+    }
+
+    if (errorRate > 0.1) {
+      return {
+        status: "warning",
+        message: `Elevated error rate: ${(errorRate * 100).toFixed(1)}%. Monitor closely.`,
+        shouldContinue: true,
+      };
+    }
+
+    return {
+      status: "healthy",
+      message: `Import progressing normally: ${(progress * 100).toFixed(1)}% complete, ${(errorRate * 100).toFixed(1)}% error rate.`,
+      shouldContinue: true,
+    };
+  }
+}