appwrite-utils-cli 1.9.7 → 1.12.0

package/src/migrations/services/ImportOrchestrator.ts

@@ -1,666 +0,0 @@
-import {
-  ID,
-  Query,
-  type Databases,
-  type Models,
-  type Storage,
-} from "node-appwrite";
-import type {
-  AppwriteConfig,
-  ConfigCollection,
-  ConfigDatabase,
-  CollectionCreate,
-  ImportDef,
-} from "appwrite-utils";
-import path from "path";
-import fs from "fs";
-import { DataTransformationService } from "./DataTransformationService.js";
-import { RateLimitManager, type RateLimitConfig } from "./RateLimitManager.js";
-import { FileHandlerService } from "./FileHandlerService.js";
-import { UserMappingService } from "./UserMappingService.js";
-import { ValidationService } from "./ValidationService.js";
-import { RelationshipResolver, type CollectionImportData } from "./RelationshipResolver.js";
-import type { ImportDataActions } from "../importDataActions.js";
-import type { SetupOptions } from "../../utilsController.js";
-import { UsersController } from "../../users/methods.js";
-import { logger } from "../../shared/logging.js";
-import { MessageFormatter } from "../../shared/messageFormatter.js";
-import { ProgressManager } from "../../shared/progressManager.js";
-import { tryAwaitWithRetry } from "../../utils/index.js";
-import { updateOperation, findOrCreateOperation } from "../../shared/migrationHelpers.js";
-import { LegacyAdapter } from "../../adapters/LegacyAdapter.js";
-import { resolveAndUpdateRelationships } from "../relationships.js";
-
-// Enhanced rate limiting configuration - now managed by RateLimitManager
-
-/**
- * Orchestrator for the entire import process.
- * Coordinates all services while preserving existing functionality and performance characteristics.
- * 
- * This replaces the monolithic ImportController and DataLoader with a cleaner, modular architecture.
- */
-export class ImportOrchestrator {
-  // Core dependencies
-  private config: AppwriteConfig;
-  private database: Databases;
-  private storage: Storage;
-  private appwriteFolderPath: string;
-  private setupOptions: SetupOptions;
-  private databasesToRun: Models.Database[];
-
-  // Services
-  private dataTransformationService: DataTransformationService;
-  private fileHandlerService: FileHandlerService;
-  private userMappingService: UserMappingService;
-  private validationService: ValidationService;
-  private relationshipResolver: RelationshipResolver;
-  private rateLimitManager: RateLimitManager;
-
-  // Import state
-  private importMap = new Map<string, CollectionImportData>();
-  private collectionImportOperations = new Map<string, string>();
-  private hasImportedUsers = false;
-  private batchLimit: number = 50; // Preserve existing batch size
-
-  constructor(
-    config: AppwriteConfig,
-    database: Databases,
-    storage: Storage,
-    appwriteFolderPath: string,
-    importDataActions: ImportDataActions,
-    setupOptions: SetupOptions,
-    databasesToRun?: Models.Database[],
-    rateLimitConfig?: Partial<RateLimitConfig>
-  ) {
-    this.config = config;
-    this.database = database;
-    this.storage = storage;
-    this.appwriteFolderPath = appwriteFolderPath;
-    this.setupOptions = setupOptions;
-    this.databasesToRun = databasesToRun || [];
-
-    // Initialize services
-    this.rateLimitManager = new RateLimitManager(rateLimitConfig);
-    this.dataTransformationService = new DataTransformationService(importDataActions);
-    this.fileHandlerService = new FileHandlerService(appwriteFolderPath, config, importDataActions, this.rateLimitManager);
-    this.userMappingService = new UserMappingService(config, this.dataTransformationService);
-    this.validationService = new ValidationService(importDataActions);
-    this.relationshipResolver = new RelationshipResolver(config, this.userMappingService);
-  }
-
-  /**
-   * Main entry point for the import process.
-   * Preserves existing import flow while using the new modular architecture.
-   */
-  async run(specificCollections?: string[]): Promise<void> {
-    let databasesToProcess: Models.Database[];
-
-    if (this.databasesToRun.length > 0) {
-      databasesToProcess = this.databasesToRun;
-    } else {
-      const allDatabases = await this.database.list();
-      databasesToProcess = allDatabases.databases;
-    }
-
-    let processedDatabase: Models.Database | undefined;
-
-    for (const db of databasesToProcess) {
-      MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
-
-      if (!processedDatabase) {
-        processedDatabase = db;
-        await this.performDatabaseImport(db, specificCollections);
-      } else if (processedDatabase.$id !== db.$id) {
-        await this.transferDataBetweenDatabases(processedDatabase, db);
-      }
-
-      console.log(`---------------------------------`);
-      console.log(`Finished import data for database: ${db.name}`);
-      console.log(`---------------------------------`);
-    }
-  }
-
-  /**
-   * Performs the complete import process for a single database.
-   */
-  private async performDatabaseImport(
-    db: Models.Database,
-    specificCollections?: string[]
-  ): Promise<void> {
-    try {
-      // Step 1: Setup and validation
-      await this.setupImportMaps(db.$id);
-      await this.loadExistingUsers();
-      
-      // Step 2: Pre-import validation
-      const validationResult = this.validationService.performPreImportValidation(
-        this.config.collections || [],
-        this.appwriteFolderPath
-      );
-      
-      if (!validationResult.isValid) {
-        logger.error("Pre-import validation failed:");
-        validationResult.errors.forEach(error => logger.error(`  - ${error}`));
-        throw new Error("Import validation failed");
-      }
-
-      if (validationResult.warnings.length > 0) {
-        logger.warn("Pre-import validation warnings:");
-        validationResult.warnings.forEach(warning => logger.warn(`  - ${warning}`));
-      }
-
-      // Step 3: Load and prepare data
-      await this.loadAndPrepareData(db, specificCollections);
-
-      // Step 4: Resolve relationships
-      logger.info("Resolving relationships...");
-      this.relationshipResolver.updateOldReferencesForNew(
-        this.importMap,
-        this.config.collections || []
-      );
-
-      // Step 5: Import collections
-      await this.importCollections(db, specificCollections);
-
-      // Step 6: Resolve and update relationships (existing logic)
-      await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-
-      // Step 7: Execute post-import actions
-      await this.executePostImportActions(db.$id, specificCollections);
-
-    } catch (error) {
-      logger.error(`Error during database import for ${db.name}:`, error);
-      throw error;
-    }
-  }
-
-  /**
-   * Sets up import maps and operation tracking.
-   * Preserves existing setup logic from DataLoader.
-   */
-  private async setupImportMaps(dbId: string): Promise<void> {
-    // Initialize the users collection in the import map
-    this.importMap.set(this.getCollectionKey("users"), { data: [] });
-
-    for (const db of this.config.databases) {
-      if (db.$id !== dbId) continue;
-      if (!this.config.collections) continue;
-
-      for (let index = 0; index < this.config.collections.length; index++) {
-        const collectionConfig = this.config.collections[index];
-        const collection = { ...collectionConfig } as CollectionCreate;
-
-        // Check if the collection exists in the database (existing logic)
-        const existingCollection = await this.findExistingCollection(db.$id, collection);
-        if (!existingCollection) {
-          logger.error(`No collection found for ${collection.name}`);
-          continue;
-        }
-
-        // Update the collection ID with the existing one
-        collectionConfig.$id = existingCollection.$id;
-        collection.$id = existingCollection.$id;
-        this.config.collections[index] = collectionConfig;
-
-        // Find or create an import operation for the collection
-        const adapter = new LegacyAdapter(this.database.client);
-        const collectionImportOperation = await findOrCreateOperation(
-          adapter,
-          dbId,
-          "importData",
-          collection.$id!
-        );
-        this.collectionImportOperations.set(
-          this.getCollectionKey(collection.name),
-          collectionImportOperation.$id
-        );
-
-        // Initialize the collection in the import map
-        this.importMap.set(this.getCollectionKey(collection.name), {
-          collection: collection,
-          data: [],
-        });
-      }
-    }
-  }
-
-  /**
-   * Loads existing users and initializes user mapping service.
-   */
-  private async loadExistingUsers(): Promise<void> {
-    const users = new UsersController(this.config, this.database);
-    const allUsers = await users.getAllUsers();
-    
-    // Initialize user mapping service with existing users
-    this.userMappingService.initializeWithExistingUsers(allUsers);
-
-    // Add existing users to import map (preserve existing logic)
-    const usersImportData = this.importMap.get(this.getCollectionKey("users"));
-    if (usersImportData) {
-      for (const user of allUsers) {
-        usersImportData.data.push({
-          finalData: {
-            ...user,
-            email: user.email?.toLowerCase(),
-            userId: user.$id,
-            docId: user.$id,
-          },
-          context: {
-            ...user,
-            email: user.email?.toLowerCase(),
-            userId: user.$id,
-            docId: user.$id,
-          },
-          rawData: user,
-        });
-      }
-      this.importMap.set(this.getCollectionKey("users"), usersImportData);
-    }
-
-    logger.info(`Loaded ${allUsers.length} existing users for deduplication`);
-  }
-
-  /**
-   * Loads and prepares data for all collections.
-   */
-  private async loadAndPrepareData(
-    db: ConfigDatabase,
-    specificCollections?: string[]
-  ): Promise<void> {
-    const collectionsToProcess = specificCollections || 
-      (this.config.collections ? this.config.collections.map(c => c.name) : []);
-
-    for (const collectionConfig of this.config.collections || []) {
-      if (!collectionsToProcess.includes(collectionConfig.name)) continue;
-      if (!collectionConfig.importDefs || collectionConfig.importDefs.length === 0) continue;
-
-      const isUsersCollection = this.userMappingService.isUsersCollection(collectionConfig.name);
-
-      // Process create definitions
-      const createDefs = collectionConfig.importDefs.filter(
-        (def: ImportDef) => def.type === "create" || !def.type
-      );
-
-      for (const createDef of createDefs) {
-        if (isUsersCollection && createDef.createUsers) {
-          await this.prepareUserCollectionData(db, collectionConfig, createDef);
-        } else {
-          await this.prepareCollectionData(db, collectionConfig, createDef);
-        }
-      }
-
-      // Process update definitions
-      const updateDefs = collectionConfig.importDefs.filter(
-        (def: ImportDef) => def.type === "update"
-      );
-
-      for (const updateDef of updateDefs) {
-        await this.prepareUpdateData(db, collectionConfig, updateDef);
-      }
-    }
-  }
-
-  /**
-   * Prepares data for a regular collection.
-   * Uses the DataTransformationService for all transformations.
-   */
-  private async prepareCollectionData(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    importDef: ImportDef
-  ): Promise<void> {
-    const rawData = this.loadDataFromFile(importDef);
-    if (rawData.length === 0) return;
-
-    await this.updateOperationStatus(db, collection, "ready", rawData.length);
-
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData) {
-      logger.error(`No collection data found for ${collection.name}`);
-      return;
-    }
-
-    for (const item of rawData) {
-      try {
-        // Generate unique ID
-        const itemId = this.generateUniqueId();
-        
-        // Create context
-        const context = this.dataTransformationService.createContext(db, collection, item, itemId);
-        
-        // Transform data
-        const transformedData = this.dataTransformationService.transformData(
-          item,
-          importDef.attributeMappings
-        );
-
-        // Validate transformed data
-        const isValid = this.dataTransformationService.validateTransformedData(
-          transformedData,
-          importDef.attributeMappings,
-          context
-        );
-
-        if (!isValid) {
-          logger.warn(`Skipping invalid item: ${JSON.stringify(item, null, 2)}`);
-          continue;
-        }
-
-        // Handle file mappings
-        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
-          importDef.attributeMappings,
-          context,
-          transformedData
-        );
-
-        // Store ID mapping if primary key exists
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
-          if (this.relationshipResolver.hasIdMapping(collection.name, oldId)) {
-            logger.error(`Duplicate primary key ${oldId} in collection ${collection.name}`);
-            continue;
-          }
-          this.relationshipResolver.setIdMapping(collection.name, oldId, itemId);
-        }
-
-        // Add to collection data
-        collectionData.data.push({
-          rawData: item,
-          context: { ...context, ...transformedData },
-          importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-          finalData: transformedData,
-        });
-
-      } catch (error) {
-        logger.error(`Error preparing item for collection ${collection.name}:`, error);
-        continue;
-      }
-    }
-
-    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-  }
-
-  /**
-   * Prepares data for user collection with deduplication.
-   * Uses the UserMappingService for sophisticated user handling.
-   */
-  private async prepareUserCollectionData(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    importDef: ImportDef
-  ): Promise<void> {
-    const rawData = this.loadDataFromFile(importDef);
-    if (rawData.length === 0) return;
-
-    await this.updateOperationStatus(db, collection, "ready", rawData.length);
-
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData) return;
-
-    for (const item of rawData) {
-      try {
-        const proposedId = this.userMappingService.getTrueUniqueUserId(collection.name);
-        
-        // Prepare user data with deduplication
-        const { transformedItem, existingId, userData } = this.userMappingService.prepareUserData(
-          item,
-          importDef.attributeMappings,
-          importDef.primaryKeyField,
-          proposedId
-        );
-
-        const finalId = existingId || proposedId;
-        const context = this.dataTransformationService.createContext(db, collection, item, finalId);
-
-        // Handle file mappings
-        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
-          importDef.attributeMappings,
-          context,
-          transformedItem
-        );
-
-        // Store ID mapping
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
-          this.relationshipResolver.setIdMapping(collection.name, oldId, finalId);
-        }
-
-        // Check for existing data and merge if needed
-        const existingDataIndex = collectionData.data.findIndex(data =>
-          data.finalData.docId === finalId || data.finalData.userId === finalId
-        );
-
-        if (existingDataIndex >= 0) {
-          // Merge with existing data
-          const existingData = collectionData.data[existingDataIndex];
-          existingData.finalData = this.dataTransformationService.mergeObjects(
-            existingData.finalData,
-            transformedItem
-          );
-          existingData.context = this.dataTransformationService.mergeObjects(
-            existingData.context,
-            { ...context, ...transformedItem, ...userData.finalData }
-          );
-        } else {
-          // Add new data
-          collectionData.data.push({
-            rawData: item,
-            context: { ...context, ...transformedItem, ...userData.finalData },
-            importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-            finalData: transformedItem,
-          });
-        }
-
-      } catch (error) {
-        logger.error(`Error preparing user data for collection ${collection.name}:`, error);
-        continue;
-      }
-    }
-
-    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-  }
-
-  /**
-   * Imports collections with rate limiting and batch processing.
-   * Preserves existing import logic with enhanced error handling.
-   */
-  private async importCollections(
-    db: ConfigDatabase,
-    specificCollections?: string[]
-  ): Promise<void> {
-    const collectionsToImport = specificCollections ||
-      (this.config.collections ? this.config.collections.map(c => c.name) : []);
-
-    for (const collection of this.config.collections || []) {
-      if (!collectionsToImport.includes(collection.name)) continue;
-
-      const isUsersCollection = this.userMappingService.isUsersCollection(collection.name);
-      
-      // Handle users collection first if needed
-      if (isUsersCollection && !this.hasImportedUsers) {
-        await this.importUsersCollection();
-      }
-
-      await this.importSingleCollection(db, collection);
-    }
-  }
-
-  /**
-   * Imports a single collection with batching and rate limiting.
-   */
-  private async importSingleCollection(
-    db: ConfigDatabase,
-    collection: CollectionCreate
-  ): Promise<void> {
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData || collectionData.data.length === 0) {
-      logger.info(`No data to import for collection: ${collection.name}`);
-      return;
-    }
-
-    logger.info(`Importing collection: ${collection.name} (${collectionData.data.length} items)`);
-
-    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-    const adapter = new LegacyAdapter(this.database.client);
-    if (operationId) {
-      await updateOperation(adapter, db.$id, operationId, { status: "in_progress" });
-    }
-
-    // Create batches for processing
-    const batches = this.createBatches(collectionData.data, this.batchLimit);
-    let processedItems = 0;
-
-    for (let i = 0; i < batches.length; i++) {
-      const batch = batches[i];
-      logger.info(`Processing batch ${i + 1} of ${batches.length} (${batch.length} items)`);
-
-      // Process batch with rate limiting
-      const batchPromises = batch.map((item, index) =>
-        this.rateLimitManager.dataInsertion(() => this.importSingleItem(db, collection, item))
-      );
-
-      const results = await Promise.allSettled(batchPromises);
-      
-      // Count successful imports
-      const successCount = results.filter(r => r.status === "fulfilled").length;
-      processedItems += successCount;
-
-      logger.info(`Batch ${i + 1} completed: ${successCount}/${batch.length} items imported`);
-
-      // Update operation progress
-      if (operationId) {
-        await updateOperation(adapter, db.$id, operationId, { progress: processedItems });
-      }
-    }
-
-    // Mark operation as completed
-    if (operationId) {
-      await updateOperation(adapter, db.$id, operationId, { status: "completed" });
-    }
-
-    logger.info(`Completed importing collection: ${collection.name} (${processedItems} items)`);
-  }
-
-  /**
-   * Imports a single item with error handling.
-   */
-  private async importSingleItem(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    item: any
-  ): Promise<void> {
-    try {
-      const id = item.finalData.docId || item.finalData.userId || item.context.docId || item.context.userId;
-
-      // Clean up internal fields
-      const cleanedData = { ...item.finalData };
-      delete cleanedData.userId;
-      delete cleanedData.docId;
-
-      if (!cleanedData || Object.keys(cleanedData).length === 0) {
-        return;
-      }
-
-      await tryAwaitWithRetry(
-        async () => await this.database.createDocument(db.$id, collection.$id!, id, cleanedData)
-      );
-
-    } catch (error) {
-      logger.error(`Error importing item to collection ${collection.name}:`, error);
-      throw error;
-    }
-  }
-
-  /**
-   * Helper method to generate consistent collection keys.
-   */
-  private getCollectionKey(name: string): string {
-    return name.toLowerCase().replace(" ", "");
-  }
-
-  /**
-   * Loads data from file based on import definition.
-   */
-  private loadDataFromFile(importDef: ImportDef): any[] {
-    try {
-      const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
-      
-      if (!fs.existsSync(filePath)) {
-        logger.error(`Import file not found: ${filePath}`);
-        return [];
-      }
-
-      const rawData = fs.readFileSync(filePath, "utf8");
-      const parsedData = importDef.basePath
-        ? JSON.parse(rawData)[importDef.basePath]
-        : JSON.parse(rawData);
-
-      logger.info(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
-      return parsedData || [];
-
-    } catch (error) {
-      logger.error(`Error loading data from file ${importDef.filePath}:`, error);
-      return [];
-    }
-  }
-
-  /**
-   * Creates batches for processing with the specified batch size.
-   */
-  private createBatches<T>(data: T[], batchSize: number): T[][] {
-    const batches: T[][] = [];
-    for (let i = 0; i < data.length; i += batchSize) {
-      batches.push(data.slice(i, i + batchSize));
-    }
-    return batches;
-  }
-
-  /**
-   * Generates a unique ID for documents.
-   */
-  private generateUniqueId(): string {
-    return ID.unique();
-  }
-
-  // Additional helper methods...
-  private async findExistingCollection(dbId: string, collection: CollectionCreate): Promise<any> {
-    // Implementation to find existing collection (preserve existing logic)
-    try {
-      const collections = await this.database.listCollections(dbId);
-      return collections.collections.find(c => c.name === collection.name || c.$id === collection.$id);
-    } catch (error) {
-      logger.error(`Error finding collection ${collection.name}:`, error);
-      return null;
-    }
-  }
-
-  private async updateOperationStatus(db: ConfigDatabase, collection: CollectionCreate, status: string, total?: number): Promise<void> {
-    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-    if (operationId) {
-      const updateData = total ? { status, total } : { status };
-      const adapter = new LegacyAdapter(this.database.client);
-      await updateOperation(adapter, db.$id, operationId, updateData);
-    }
-  }
-
-  private async importUsersCollection(): Promise<void> {
-    // Implementation for importing users collection (preserve existing logic)
-    // This would handle the sophisticated user import logic
-    this.hasImportedUsers = true;
-  }
-
-  private async prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): Promise<void> {
-    // Implementation for preparing update data (preserve existing logic)
-    // This would handle the update logic from the original DataLoader
-  }
-
-  private async executePostImportActions(dbId: string, specificCollections?: string[]): Promise<void> {
-    // Implementation for executing post-import actions (preserve existing logic)
-    // This would handle file uploads and other post-import actions
-  }
-
-  private async transferDataBetweenDatabases(sourceDb: Models.Database, targetDb: Models.Database): Promise<void> {
-    // Implementation for transferring data between databases (preserve existing logic)
-    // This would handle the existing transfer logic
-  }
-}