appwrite-utils-cli 1.9.7 → 1.12.0

package/src/migrations/services/UserMappingService.ts

@@ -1,345 +0,0 @@
-import { z } from "zod";
-import { ID } from "node-appwrite";
-import type {
-  AttributeMappings,
-  AppwriteConfig,
-} from "appwrite-utils";
-import { AuthUserCreateSchema } from "../../schemas/authUser.js";
-import { logger } from "../../shared/logging.js";
-import type { DataTransformationService } from "./DataTransformationService.js";
-
-/**
- * Service responsible for user mapping and deduplication during import.
- * Preserves all existing sophisticated user handling logic from DataLoader.
- * Extracted to provide focused, testable user management operations.
- */
-export class UserMappingService {
-  private config: AppwriteConfig;
-  private dataTransformationService: DataTransformationService;
-  
-  // Maps to hold email and phone to user ID mappings for uniqueness in User Accounts
-  private emailToUserIdMap = new Map<string, string>();
-  private phoneToUserIdMap = new Map<string, string>();
-  private userIdSet = new Set<string>();
-  
-  // Map to hold the merged user map for relationship resolution
-  // Will hold an array of the old user ID's that are mapped to the same new user ID
-  // For example, if there are two users with the same email, they will both be mapped to the same new user ID
-  // Prevents duplicate users with the other two maps below it and allows me to keep the old ID's
-  private mergedUserMap = new Map<string, string[]>();
-  
-  // Map to track existing users
-  public userExistsMap = new Map<string, boolean>();
-
-  constructor(
-    config: AppwriteConfig,
-    dataTransformationService: DataTransformationService
-  ) {
-    this.config = config;
-    this.dataTransformationService = dataTransformationService;
-  }
-
-  /**
-   * Initializes user maps with existing users from the system.
-   * Preserves existing user loading logic from DataLoader.
-   * 
-   * @param existingUsers - Array of existing users from the system
-   */
-  initializeWithExistingUsers(existingUsers: any[]): void {
-    // Clear existing maps
-    this.emailToUserIdMap.clear();
-    this.phoneToUserIdMap.clear();
-    this.userIdSet.clear();
-    this.userExistsMap.clear();
-
-    // Iterate over the users and setup our maps ahead of time for email and phone
-    for (const user of existingUsers) {
-      if (user.email) {
-        this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
-      }
-      if (user.phone) {
-        this.phoneToUserIdMap.set(user.phone, user.$id);
-      }
-      this.userExistsMap.set(user.$id, true);
-      this.userIdSet.add(user.$id);
-    }
-
-    logger.info(`Initialized user maps with ${existingUsers.length} existing users`);
-  }
-
-  /**
-   * Generates a unique ID that doesn't conflict with existing user IDs.
-   * Preserves existing unique ID generation logic from DataLoader.
-   * 
-   * @param collectionName - The collection name for context
-   * @returns A truly unique ID
-   */
-  getTrueUniqueUserId(collectionName: string): string {
-    let newId = ID.unique();
-    let condition =
-      this.userExistsMap.has(newId) ||
-      this.userIdSet.has(newId) ||
-      Array.from(this.emailToUserIdMap.values()).includes(newId) ||
-      Array.from(this.phoneToUserIdMap.values()).includes(newId);
-      
-    while (condition) {
-      newId = ID.unique();
-      condition =
-        this.userExistsMap.has(newId) ||
-        this.userIdSet.has(newId) ||
-        Array.from(this.emailToUserIdMap.values()).includes(newId) ||
-        Array.from(this.phoneToUserIdMap.values()).includes(newId);
-    }
-    return newId;
-  }
-
-  /**
-   * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
-   * and then returning the transformed item without user-specific keys.
-   * 
-   * Preserves existing sophisticated user deduplication logic from DataLoader.
-   *
-   * @param item - The raw item to be processed.
-   * @param attributeMappings - The attribute mappings for the item.
-   * @param primaryKeyField - The primary key field name
-   * @param newId - The proposed new ID for the user
-   * @returns Object containing transformed item, existing ID if duplicate, and user data
-   */
-  prepareUserData(
-    item: any,
-    attributeMappings: AttributeMappings,
-    primaryKeyField: string,
-    newId: string
-  ): {
-    transformedItem: any;
-    existingId: string | undefined;
-    userData: {
-      rawData: any;
-      finalData: z.infer<typeof AuthUserCreateSchema>;
-    };
-  } {
-    // Ensure we have a truly unique ID
-    if (
-      this.userIdSet.has(newId) ||
-      this.userExistsMap.has(newId) ||
-      Array.from(this.emailToUserIdMap.values()).includes(newId) ||
-      Array.from(this.phoneToUserIdMap.values()).includes(newId)
-    ) {
-      newId = this.getTrueUniqueUserId("users");
-    }
-
-    let transformedItem = this.dataTransformationService.transformData(item, attributeMappings);
-    let userData = AuthUserCreateSchema.safeParse(transformedItem);
-    
-    if (userData.data?.email) {
-      userData.data.email = userData.data.email.toLowerCase();
-    }
-    
-    if (!userData.success || !(userData.data.email || userData.data.phone)) {
-      logger.error(
-        `Invalid user data: ${JSON.stringify(
-          userData.error?.issues,
-          undefined,
-          2
-        )} or missing email/phone`
-      );
-
-      const userKeys = ["email", "phone", "name", "labels", "prefs"];
-      userKeys.forEach((key) => {
-        if (transformedItem.hasOwnProperty(key)) {
-          delete transformedItem[key];
-        }
-      });
-      
-      return {
-        transformedItem,
-        existingId: undefined,
-        userData: {
-          rawData: item,
-          finalData: transformedItem,
-        },
-      };
-    }
-
-    const email = userData.data.email?.toLowerCase();
-    const phone = userData.data.phone;
-    let existingId: string | undefined;
-
-    // Check for duplicate email and phone
-    if (email && this.emailToUserIdMap.has(email)) {
-      existingId = this.emailToUserIdMap.get(email);
-      if (phone && !this.phoneToUserIdMap.has(phone)) {
-        this.phoneToUserIdMap.set(phone, newId);
-      }
-    } else if (phone && this.phoneToUserIdMap.has(phone)) {
-      existingId = this.phoneToUserIdMap.get(phone);
-      if (email && !this.emailToUserIdMap.has(email)) {
-        this.emailToUserIdMap.set(email, newId);
-      }
-    } else {
-      if (email) this.emailToUserIdMap.set(email, newId);
-      if (phone) this.phoneToUserIdMap.set(phone, newId);
-    }
-
-    if (existingId) {
-      userData.data.userId = existingId;
-      const mergedUsers = this.mergedUserMap.get(existingId) || [];
-      mergedUsers.push(`${item[primaryKeyField]}`);
-      this.mergedUserMap.set(existingId, mergedUsers);
-
-      const userKeys = ["email", "phone", "name", "labels", "prefs"];
-      userKeys.forEach((key) => {
-        if (transformedItem.hasOwnProperty(key)) {
-          delete transformedItem[key];
-        }
-      });
-      
-      return {
-        transformedItem: {
-          ...transformedItem,
-          userId: existingId,
-          docId: existingId,
-        },
-        existingId,
-        userData: {
-          rawData: item,
-          finalData: userData.data,
-        },
-      };
-    } else {
-      existingId = newId;
-      userData.data.userId = existingId;
-    }
-
-    const userKeys = ["email", "phone", "name", "labels", "prefs"];
-    userKeys.forEach((key) => {
-      if (transformedItem.hasOwnProperty(key)) {
-        delete transformedItem[key];
-      }
-    });
-
-    this.userIdSet.add(existingId);
-
-    return {
-      transformedItem: {
-        ...transformedItem,
-        userId: existingId,
-        docId: existingId,
-      },
-      existingId,
-      userData: {
-        rawData: item,
-        finalData: userData.data,
-      },
-    };
-  }
-
-  /**
-   * Checks if a collection is the users collection based on configuration.
-   * 
-   * @param collectionName - The collection name to check
-   * @returns True if this is the users collection
-   */
-  isUsersCollection(collectionName: string): boolean {
-    return !!(
-      this.config.usersCollectionName &&
-      this.getCollectionKey(this.config.usersCollectionName) ===
-        this.getCollectionKey(collectionName)
-    );
-  }
-
-  /**
-   * Helper method to generate a consistent key for collections.
-   * Preserves existing logic from DataLoader.
-   */
-  private getCollectionKey(name: string): string {
-    return name.toLowerCase().replace(" ", "");
-  }
-
-  /**
-   * Gets merged user mappings for relationship resolution.
-   * 
-   * @returns Map of merged user IDs to arrays of original IDs
-   */
-  getMergedUserMap(): Map<string, string[]> {
-    return new Map(this.mergedUserMap);
-  }
-
-  /**
-   * Finds the new user ID for an old user ID, considering merged users.
-   * Preserves existing merged user lookup logic from DataLoader.
-   * 
-   * @param oldId - The old user ID to look up
-   * @returns The new user ID if found, otherwise the original ID
-   */
-  findNewUserIdForOldId(oldId: string): string {
-    // Check if this old ID was merged into another user
-    for (const [newUserId, oldUserIds] of this.mergedUserMap.entries()) {
-      if (oldUserIds.includes(`${oldId}`)) {
-        return newUserId;
-      }
-    }
-    
-    // If not found in merged users, return the original ID
-    return oldId;
-  }
-
-  /**
-   * Merges user data from duplicate entries.
-   * Preserves existing user data merging logic.
-   * 
-   * @param existingUserData - The existing user data
-   * @param newUserData - The new user data to merge
-   * @returns Merged user data
-   */
-  mergeUserData(existingUserData: any, newUserData: any): any {
-    return this.dataTransformationService.mergeObjects(existingUserData, newUserData);
-  }
-
-  /**
-   * Gets statistics about user mapping operations.
-   * Useful for import planning and reporting.
-   * 
-   * @returns User mapping statistics
-   */
-  getStatistics(): {
-    totalExistingUsers: number;
-    emailMappings: number;
-    phoneMappings: number;
-    mergedUsers: number;
-    totalMergedOldIds: number;
-  } {
-    let totalMergedOldIds = 0;
-    for (const oldIds of this.mergedUserMap.values()) {
-      totalMergedOldIds += oldIds.length;
-    }
-
-    return {
-      totalExistingUsers: this.userExistsMap.size,
-      emailMappings: this.emailToUserIdMap.size,
-      phoneMappings: this.phoneToUserIdMap.size,
-      mergedUsers: this.mergedUserMap.size,
-      totalMergedOldIds,
-    };
-  }
-
-  /**
-   * Validates user mapping configuration.
-   * Ensures the user mapping setup is correct before import.
-   * 
-   * @returns Array of validation errors (empty if valid)
-   */
-  validateConfiguration(): string[] {
-    const errors: string[] = [];
-
-    if (!this.config.usersCollectionName) {
-      errors.push("usersCollectionName is not configured");
-    }
-
-    if (!this.config.documentBucketId) {
-      errors.push("documentBucketId is not configured");
-    }
-
-    return errors;
-  }
-}

package/src/migrations/services/ValidationService.ts

@@ -1,349 +0,0 @@
-import type {
-  AttributeMappings,
-  ImportDef,
-  CollectionCreate,
-} from "appwrite-utils";
-import type { ImportDataActions } from "../importDataActions.js";
-import { logger } from "../../shared/logging.js";
-import fs from "fs";
-import path from "path";
-
-/**
- * Service responsible for validation during import operations.
- * Provides centralized validation logic extracted from ImportDataActions and DataLoader.
- */
-export class ValidationService {
-  private importDataActions: ImportDataActions;
-
-  constructor(importDataActions: ImportDataActions) {
-    this.importDataActions = importDataActions;
-  }
-
-  /**
-   * Validates a single data item based on defined validation rules.
-   * Preserves existing validation logic from ImportDataActions.
-   * 
-   * @param item - The data item to validate
-   * @param attributeMappings - The attribute mappings for the data item
-   * @param context - The context for resolving templated parameters in validation rules
-   * @returns True if the item is valid, false otherwise
-   */
-  validateDataItem(
-    item: any,
-    attributeMappings: AttributeMappings,
-    context: { [key: string]: any }
-  ): boolean {
-    try {
-      return this.importDataActions.validateItem(item, attributeMappings, context);
-    } catch (error) {
-      logger.error(`Validation error for item: ${JSON.stringify(item, null, 2)}`, error);
-      return false;
-    }
-  }
-
-  /**
-   * Validates an import definition before processing begins.
-   * Provides early validation to catch configuration issues.
-   * 
-   * @param importDef - The import definition to validate
-   * @returns Array of validation errors (empty if valid)
-   */
-  validateImportDefinition(importDef: ImportDef): string[] {
-    const errors: string[] = [];
-
-    // Validate required fields
-    if (!importDef.filePath) {
-      errors.push("filePath is required");
-    }
-
-    if (!importDef.attributeMappings || importDef.attributeMappings.length === 0) {
-      errors.push("attributeMappings are required");
-    }
-
-    // Validate attribute mappings
-    if (importDef.attributeMappings) {
-      for (let i = 0; i < importDef.attributeMappings.length; i++) {
-        const mapping = importDef.attributeMappings[i];
-        
-        if (!mapping.targetKey) {
-          errors.push(`attributeMappings[${i}]: targetKey is required`);
-        }
-
-        if (!mapping.oldKey && !mapping.oldKeys && mapping.valueToSet === undefined) {
-          errors.push(
-            `attributeMappings[${i}]: must have either oldKey, oldKeys, or valueToSet`
-          );
-        }
-
-        // Validate file data if present
-        if (mapping.fileData) {
-          if (!mapping.fileData.path) {
-            errors.push(`attributeMappings[${i}].fileData: path is required`);
-          }
-          if (!mapping.fileData.name) {
-            errors.push(`attributeMappings[${i}].fileData: name is required`);
-          }
-        }
-
-        // Validate converters if present
-        if (mapping.converters && !Array.isArray(mapping.converters)) {
-          errors.push(`attributeMappings[${i}].converters: must be an array`);
-        }
-
-        // Validate validation actions if present
-        if (mapping.validationActions) {
-          if (!Array.isArray(mapping.validationActions)) {
-            errors.push(`attributeMappings[${i}].validationActions: must be an array`);
-          } else {
-            for (let j = 0; j < mapping.validationActions.length; j++) {
-              const action = mapping.validationActions[j];
-              if (!action.action) {
-                errors.push(
-                  `attributeMappings[${i}].validationActions[${j}]: action is required`
-                );
-              }
-              if (!Array.isArray(action.params)) {
-                errors.push(
-                  `attributeMappings[${i}].validationActions[${j}]: params must be an array`
-                );
-              }
-            }
-          }
-        }
-
-        // Validate post-import actions if present
-        if (mapping.postImportActions) {
-          if (!Array.isArray(mapping.postImportActions)) {
-            errors.push(`attributeMappings[${i}].postImportActions: must be an array`);
-          } else {
-            for (let j = 0; j < mapping.postImportActions.length; j++) {
-              const action = mapping.postImportActions[j];
-              if (!action.action) {
-                errors.push(
-                  `attributeMappings[${i}].postImportActions[${j}]: action is required`
-                );
-              }
-              if (!Array.isArray(action.params)) {
-                errors.push(
-                  `attributeMappings[${i}].postImportActions[${j}]: params must be an array`
-                );
-              }
-            }
-          }
-        }
-      }
-    }
-
-    // Validate ID mappings if present
-    if (importDef.idMappings) {
-      for (let i = 0; i < importDef.idMappings.length; i++) {
-        const idMapping = importDef.idMappings[i];
-        
-        if (!idMapping.sourceField) {
-          errors.push(`idMappings[${i}]: sourceField is required`);
-        }
-        if (!idMapping.targetField) {
-          errors.push(`idMappings[${i}]: targetField is required`);
-        }
-        if (!idMapping.targetCollection) {
-          errors.push(`idMappings[${i}]: targetCollection is required`);
-        }
-      }
-    }
-
-    // Validate update mapping if present
-    if (importDef.updateMapping) {
-      if (!importDef.updateMapping.originalIdField) {
-        errors.push("updateMapping: originalIdField is required");
-      }
-      if (!importDef.updateMapping.targetField) {
-        errors.push("updateMapping: targetField is required");
-      }
-    }
-
-    return errors;
-  }
-
-  /**
-   * Validates a collection configuration for import compatibility.
-   * Ensures the collection has the necessary attributes for import operations.
-   * 
-   * @param collection - The collection to validate
-   * @param importDefs - The import definitions that will be used with this collection
-   * @returns Array of validation errors (empty if valid)
-   */
-  validateCollectionForImport(
-    collection: CollectionCreate,
-    importDefs: ImportDef[]
-  ): string[] {
-    const errors: string[] = [];
-
-    if (!collection.name) {
-      errors.push("Collection name is required");
-    }
-
-    if (!collection.attributes || collection.attributes.length === 0) {
-      errors.push("Collection must have attributes defined");
-    }
-
-    // Validate that target keys in import definitions exist as attributes
-    for (const importDef of importDefs) {
-      for (const mapping of importDef.attributeMappings) {
-        const attributeExists = collection.attributes?.some(
-          attr => attr.key === mapping.targetKey
-        );
-        
-        if (!attributeExists) {
-          errors.push(
-            `Attribute '${mapping.targetKey}' referenced in import mapping not found in collection`
-          );
-        }
-      }
-    }
-
-    return errors;
-  }
-
-  /**
-   * Validates data consistency across multiple collections.
-   * Checks for relationship integrity and data consistency.
-   * 
-   * @param collections - Array of collections to validate
-   * @returns Array of validation errors (empty if valid)
-   */
-  validateCrossCollectionConsistency(
-    collections: CollectionCreate[]
-  ): string[] {
-    const errors: string[] = [];
-    const collectionNames = new Set(collections.map(c => c.name));
-
-    for (const collection of collections) {
-      if (!collection.importDefs) continue;
-
-      for (const importDef of collection.importDefs) {
-        if (!importDef.idMappings) continue;
-
-        for (const idMapping of importDef.idMappings) {
-          // Check if target collection exists
-          if (!collectionNames.has(idMapping.targetCollection)) {
-            errors.push(
-              `Collection '${collection.name}' references non-existent target collection '${idMapping.targetCollection}' in ID mapping`
-            );
-          }
-        }
-      }
-    }
-
-    return errors;
-  }
-
-  /**
-   * Performs comprehensive pre-import validation.
-   * Validates all aspects of the import configuration before starting.
-   * 
-   * @param collections - Collections that will be imported
-   * @param appwriteFolderPath - Path to the appwrite folder for file validation
-   * @returns Validation result with errors and warnings
-   */
-  performPreImportValidation(
-    collections: CollectionCreate[],
-    appwriteFolderPath: string
-  ): {
-    errors: string[];
-    warnings: string[];
-    isValid: boolean;
-  } {
-    const errors: string[] = [];
-    const warnings: string[] = [];
-
-    // Validate each collection
-    for (const collection of collections) {
-      const collectionErrors = this.validateCollectionForImport(
-        collection,
-        collection.importDefs || []
-      );
-      errors.push(...collectionErrors.map(err => `${collection.name}: ${err}`));
-
-      // Validate each import definition
-      if (collection.importDefs) {
-        for (let i = 0; i < collection.importDefs.length; i++) {
-          const importDef = collection.importDefs[i];
-          const importDefErrors = this.validateImportDefinition(importDef);
-          errors.push(
-            ...importDefErrors.map(
-              err => `${collection.name}.importDefs[${i}]: ${err}`
-            )
-          );
-
-          // Check if import file exists
-          try {
-            const filePath = path.resolve(appwriteFolderPath, importDef.filePath);
-            if (!fs.existsSync(filePath)) {
-              errors.push(
-                `${collection.name}.importDefs[${i}]: Import file not found: ${filePath}`
-              );
-            }
-          } catch (error) {
-            warnings.push(
-              `${collection.name}.importDefs[${i}]: Could not validate file existence: ${error}`
-            );
-          }
-        }
-      }
-    }
-
-    // Validate cross-collection consistency
-    const crossCollectionErrors = this.validateCrossCollectionConsistency(collections);
-    errors.push(...crossCollectionErrors);
-
-    return {
-      errors,
-      warnings,
-      isValid: errors.length === 0,
-    };
-  }
-
-  /**
-   * Validates import progress and data integrity during import.
-   * Can be used for periodic validation during long-running imports.
-   * 
-   * @param processedItems - Number of items processed so far
-   * @param totalItems - Total number of items to process
-   * @param errorCount - Number of errors encountered
-   * @returns Validation status with recommendations
-   */
-  validateImportProgress(
-    processedItems: number,
-    totalItems: number,
-    errorCount: number
-  ): {
-    status: "healthy" | "warning" | "critical";
-    message: string;
-    shouldContinue: boolean;
-  } {
-    const errorRate = errorCount / Math.max(processedItems, 1);
-    const progress = processedItems / totalItems;
-
-    if (errorRate > 0.5) {
-      return {
-        status: "critical",
-        message: `Error rate too high: ${(errorRate * 100).toFixed(1)}%. Consider stopping import.`,
-        shouldContinue: false,
-      };
-    }
-
-    if (errorRate > 0.1) {
-      return {
-        status: "warning",
-        message: `Elevated error rate: ${(errorRate * 100).toFixed(1)}%. Monitor closely.`,
-        shouldContinue: true,
-      };
-    }
-
-    return {
-      status: "healthy",
-      message: `Import progressing normally: ${(progress * 100).toFixed(1)}% complete, ${(errorRate * 100).toFixed(1)}% error rate.`,
-      shouldContinue: true,
-    };
-  }
-}