appwrite-utils-cli 1.9.7 → 1.11.0

package/src/migrations/services/ImportOrchestrator.ts

@@ -1,666 +1,675 @@
-import {
-  ID,
-  Query,
-  type Databases,
-  type Models,
-  type Storage,
-} from "node-appwrite";
-import type {
-  AppwriteConfig,
-  ConfigCollection,
-  ConfigDatabase,
-  CollectionCreate,
-  ImportDef,
-} from "appwrite-utils";
-import path from "path";
-import fs from "fs";
-import { DataTransformationService } from "./DataTransformationService.js";
-import { RateLimitManager, type RateLimitConfig } from "./RateLimitManager.js";
-import { FileHandlerService } from "./FileHandlerService.js";
-import { UserMappingService } from "./UserMappingService.js";
-import { ValidationService } from "./ValidationService.js";
-import { RelationshipResolver, type CollectionImportData } from "./RelationshipResolver.js";
-import type { ImportDataActions } from "../importDataActions.js";
-import type { SetupOptions } from "../../utilsController.js";
-import { UsersController } from "../../users/methods.js";
-import { logger } from "../../shared/logging.js";
-import { MessageFormatter } from "../../shared/messageFormatter.js";
-import { ProgressManager } from "../../shared/progressManager.js";
-import { tryAwaitWithRetry } from "../../utils/index.js";
-import { updateOperation, findOrCreateOperation } from "../../shared/migrationHelpers.js";
-import { LegacyAdapter } from "../../adapters/LegacyAdapter.js";
-import { resolveAndUpdateRelationships } from "../relationships.js";
-
-// Enhanced rate limiting configuration - now managed by RateLimitManager
-
-/**
- * Orchestrator for the entire import process.
- * Coordinates all services while preserving existing functionality and performance characteristics.
- * 
- * This replaces the monolithic ImportController and DataLoader with a cleaner, modular architecture.
- */
-export class ImportOrchestrator {
-  // Core dependencies
-  private config: AppwriteConfig;
-  private database: Databases;
-  private storage: Storage;
-  private appwriteFolderPath: string;
-  private setupOptions: SetupOptions;
-  private databasesToRun: Models.Database[];
-
-  // Services
-  private dataTransformationService: DataTransformationService;
-  private fileHandlerService: FileHandlerService;
-  private userMappingService: UserMappingService;
-  private validationService: ValidationService;
-  private relationshipResolver: RelationshipResolver;
-  private rateLimitManager: RateLimitManager;
-
-  // Import state
-  private importMap = new Map<string, CollectionImportData>();
-  private collectionImportOperations = new Map<string, string>();
-  private hasImportedUsers = false;
-  private batchLimit: number = 50; // Preserve existing batch size
-
-  constructor(
-    config: AppwriteConfig,
-    database: Databases,
-    storage: Storage,
-    appwriteFolderPath: string,
-    importDataActions: ImportDataActions,
-    setupOptions: SetupOptions,
-    databasesToRun?: Models.Database[],
-    rateLimitConfig?: Partial<RateLimitConfig>
-  ) {
-    this.config = config;
-    this.database = database;
-    this.storage = storage;
-    this.appwriteFolderPath = appwriteFolderPath;
-    this.setupOptions = setupOptions;
-    this.databasesToRun = databasesToRun || [];
-
-    // Initialize services
-    this.rateLimitManager = new RateLimitManager(rateLimitConfig);
-    this.dataTransformationService = new DataTransformationService(importDataActions);
-    this.fileHandlerService = new FileHandlerService(appwriteFolderPath, config, importDataActions, this.rateLimitManager);
-    this.userMappingService = new UserMappingService(config, this.dataTransformationService);
-    this.validationService = new ValidationService(importDataActions);
-    this.relationshipResolver = new RelationshipResolver(config, this.userMappingService);
-  }
-
-  /**
-   * Main entry point for the import process.
-   * Preserves existing import flow while using the new modular architecture.
-   */
-  async run(specificCollections?: string[]): Promise<void> {
-    let databasesToProcess: Models.Database[];
-
-    if (this.databasesToRun.length > 0) {
-      databasesToProcess = this.databasesToRun;
-    } else {
-      const allDatabases = await this.database.list();
-      databasesToProcess = allDatabases.databases;
-    }
-
-    let processedDatabase: Models.Database | undefined;
-
-    for (const db of databasesToProcess) {
-      MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
-
-      if (!processedDatabase) {
-        processedDatabase = db;
-        await this.performDatabaseImport(db, specificCollections);
-      } else if (processedDatabase.$id !== db.$id) {
-        await this.transferDataBetweenDatabases(processedDatabase, db);
-      }
-
-      console.log(`---------------------------------`);
-      console.log(`Finished import data for database: ${db.name}`);
-      console.log(`---------------------------------`);
-    }
-  }
-
-  /**
-   * Performs the complete import process for a single database.
-   */
-  private async performDatabaseImport(
-    db: Models.Database,
-    specificCollections?: string[]
-  ): Promise<void> {
-    try {
-      // Step 1: Setup and validation
-      await this.setupImportMaps(db.$id);
-      await this.loadExistingUsers();
-      
-      // Step 2: Pre-import validation
-      const validationResult = this.validationService.performPreImportValidation(
-        this.config.collections || [],
-        this.appwriteFolderPath
-      );
-      
-      if (!validationResult.isValid) {
-        logger.error("Pre-import validation failed:");
-        validationResult.errors.forEach(error => logger.error(`  - ${error}`));
-        throw new Error("Import validation failed");
-      }
-
-      if (validationResult.warnings.length > 0) {
-        logger.warn("Pre-import validation warnings:");
-        validationResult.warnings.forEach(warning => logger.warn(`  - ${warning}`));
-      }
-
-      // Step 3: Load and prepare data
-      await this.loadAndPrepareData(db, specificCollections);
-
-      // Step 4: Resolve relationships
-      logger.info("Resolving relationships...");
-      this.relationshipResolver.updateOldReferencesForNew(
-        this.importMap,
-        this.config.collections || []
-      );
-
-      // Step 5: Import collections
-      await this.importCollections(db, specificCollections);
-
-      // Step 6: Resolve and update relationships (existing logic)
-      await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-
-      // Step 7: Execute post-import actions
-      await this.executePostImportActions(db.$id, specificCollections);
-
-    } catch (error) {
-      logger.error(`Error during database import for ${db.name}:`, error);
-      throw error;
-    }
-  }
-
-  /**
-   * Sets up import maps and operation tracking.
-   * Preserves existing setup logic from DataLoader.
-   */
-  private async setupImportMaps(dbId: string): Promise<void> {
-    // Initialize the users collection in the import map
-    this.importMap.set(this.getCollectionKey("users"), { data: [] });
-
-    for (const db of this.config.databases) {
-      if (db.$id !== dbId) continue;
-      if (!this.config.collections) continue;
-
-      for (let index = 0; index < this.config.collections.length; index++) {
-        const collectionConfig = this.config.collections[index];
-        const collection = { ...collectionConfig } as CollectionCreate;
-
-        // Check if the collection exists in the database (existing logic)
-        const existingCollection = await this.findExistingCollection(db.$id, collection);
-        if (!existingCollection) {
-          logger.error(`No collection found for ${collection.name}`);
-          continue;
-        }
-
-        // Update the collection ID with the existing one
-        collectionConfig.$id = existingCollection.$id;
-        collection.$id = existingCollection.$id;
-        this.config.collections[index] = collectionConfig;
-
-        // Find or create an import operation for the collection
-        const adapter = new LegacyAdapter(this.database.client);
-        const collectionImportOperation = await findOrCreateOperation(
-          adapter,
-          dbId,
-          "importData",
-          collection.$id!
-        );
-        this.collectionImportOperations.set(
-          this.getCollectionKey(collection.name),
-          collectionImportOperation.$id
-        );
-
-        // Initialize the collection in the import map
-        this.importMap.set(this.getCollectionKey(collection.name), {
-          collection: collection,
-          data: [],
-        });
-      }
-    }
-  }
-
-  /**
-   * Loads existing users and initializes user mapping service.
-   */
-  private async loadExistingUsers(): Promise<void> {
-    const users = new UsersController(this.config, this.database);
-    const allUsers = await users.getAllUsers();
-    
-    // Initialize user mapping service with existing users
-    this.userMappingService.initializeWithExistingUsers(allUsers);
-
-    // Add existing users to import map (preserve existing logic)
-    const usersImportData = this.importMap.get(this.getCollectionKey("users"));
-    if (usersImportData) {
-      for (const user of allUsers) {
-        usersImportData.data.push({
-          finalData: {
-            ...user,
-            email: user.email?.toLowerCase(),
-            userId: user.$id,
-            docId: user.$id,
-          },
-          context: {
-            ...user,
-            email: user.email?.toLowerCase(),
-            userId: user.$id,
-            docId: user.$id,
-          },
-          rawData: user,
-        });
-      }
-      this.importMap.set(this.getCollectionKey("users"), usersImportData);
-    }
-
-    logger.info(`Loaded ${allUsers.length} existing users for deduplication`);
-  }
-
-  /**
-   * Loads and prepares data for all collections.
-   */
-  private async loadAndPrepareData(
-    db: ConfigDatabase,
-    specificCollections?: string[]
-  ): Promise<void> {
-    const collectionsToProcess = specificCollections || 
-      (this.config.collections ? this.config.collections.map(c => c.name) : []);
-
-    for (const collectionConfig of this.config.collections || []) {
-      if (!collectionsToProcess.includes(collectionConfig.name)) continue;
-      if (!collectionConfig.importDefs || collectionConfig.importDefs.length === 0) continue;
-
-      const isUsersCollection = this.userMappingService.isUsersCollection(collectionConfig.name);
-
-      // Process create definitions
-      const createDefs = collectionConfig.importDefs.filter(
-        (def: ImportDef) => def.type === "create" || !def.type
-      );
-
-      for (const createDef of createDefs) {
-        if (isUsersCollection && createDef.createUsers) {
-          await this.prepareUserCollectionData(db, collectionConfig, createDef);
-        } else {
-          await this.prepareCollectionData(db, collectionConfig, createDef);
-        }
-      }
-
-      // Process update definitions
-      const updateDefs = collectionConfig.importDefs.filter(
-        (def: ImportDef) => def.type === "update"
-      );
-
-      for (const updateDef of updateDefs) {
-        await this.prepareUpdateData(db, collectionConfig, updateDef);
-      }
-    }
-  }
-
-  /**
-   * Prepares data for a regular collection.
-   * Uses the DataTransformationService for all transformations.
-   */
-  private async prepareCollectionData(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    importDef: ImportDef
-  ): Promise<void> {
-    const rawData = this.loadDataFromFile(importDef);
-    if (rawData.length === 0) return;
-
-    await this.updateOperationStatus(db, collection, "ready", rawData.length);
-
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData) {
-      logger.error(`No collection data found for ${collection.name}`);
-      return;
-    }
-
-    for (const item of rawData) {
-      try {
-        // Generate unique ID
-        const itemId = this.generateUniqueId();
-        
-        // Create context
-        const context = this.dataTransformationService.createContext(db, collection, item, itemId);
-        
-        // Transform data
-        const transformedData = this.dataTransformationService.transformData(
-          item,
-          importDef.attributeMappings
-        );
-
-        // Validate transformed data
-        const isValid = this.dataTransformationService.validateTransformedData(
-          transformedData,
-          importDef.attributeMappings,
-          context
-        );
-
-        if (!isValid) {
-          logger.warn(`Skipping invalid item: ${JSON.stringify(item, null, 2)}`);
-          continue;
-        }
-
-        // Handle file mappings
-        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
-          importDef.attributeMappings,
-          context,
-          transformedData
-        );
-
-        // Store ID mapping if primary key exists
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
-          if (this.relationshipResolver.hasIdMapping(collection.name, oldId)) {
-            logger.error(`Duplicate primary key ${oldId} in collection ${collection.name}`);
-            continue;
-          }
-          this.relationshipResolver.setIdMapping(collection.name, oldId, itemId);
-        }
-
-        // Add to collection data
-        collectionData.data.push({
-          rawData: item,
-          context: { ...context, ...transformedData },
-          importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-          finalData: transformedData,
-        });
-
-      } catch (error) {
-        logger.error(`Error preparing item for collection ${collection.name}:`, error);
-        continue;
-      }
-    }
-
-    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-  }
-
-  /**
-   * Prepares data for user collection with deduplication.
-   * Uses the UserMappingService for sophisticated user handling.
-   */
-  private async prepareUserCollectionData(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    importDef: ImportDef
-  ): Promise<void> {
-    const rawData = this.loadDataFromFile(importDef);
-    if (rawData.length === 0) return;
-
-    await this.updateOperationStatus(db, collection, "ready", rawData.length);
-
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData) return;
-
-    for (const item of rawData) {
-      try {
-        const proposedId = this.userMappingService.getTrueUniqueUserId(collection.name);
-        
-        // Prepare user data with deduplication
-        const { transformedItem, existingId, userData } = this.userMappingService.prepareUserData(
-          item,
-          importDef.attributeMappings,
-          importDef.primaryKeyField,
-          proposedId
-        );
-
-        const finalId = existingId || proposedId;
-        const context = this.dataTransformationService.createContext(db, collection, item, finalId);
-
-        // Handle file mappings
-        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
-          importDef.attributeMappings,
-          context,
-          transformedItem
-        );
-
-        // Store ID mapping
-        if (importDef.primaryKeyField) {
-          const oldId = item[importDef.primaryKeyField];
-          this.relationshipResolver.setIdMapping(collection.name, oldId, finalId);
-        }
-
-        // Check for existing data and merge if needed
-        const existingDataIndex = collectionData.data.findIndex(data =>
-          data.finalData.docId === finalId || data.finalData.userId === finalId
-        );
-
-        if (existingDataIndex >= 0) {
-          // Merge with existing data
-          const existingData = collectionData.data[existingDataIndex];
-          existingData.finalData = this.dataTransformationService.mergeObjects(
-            existingData.finalData,
-            transformedItem
-          );
-          existingData.context = this.dataTransformationService.mergeObjects(
-            existingData.context,
-            { ...context, ...transformedItem, ...userData.finalData }
-          );
-        } else {
-          // Add new data
-          collectionData.data.push({
-            rawData: item,
-            context: { ...context, ...transformedItem, ...userData.finalData },
-            importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-            finalData: transformedItem,
-          });
-        }
-
-      } catch (error) {
-        logger.error(`Error preparing user data for collection ${collection.name}:`, error);
-        continue;
-      }
-    }
-
-    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-  }
-
-  /**
-   * Imports collections with rate limiting and batch processing.
-   * Preserves existing import logic with enhanced error handling.
-   */
-  private async importCollections(
-    db: ConfigDatabase,
-    specificCollections?: string[]
-  ): Promise<void> {
-    const collectionsToImport = specificCollections ||
-      (this.config.collections ? this.config.collections.map(c => c.name) : []);
-
-    for (const collection of this.config.collections || []) {
-      if (!collectionsToImport.includes(collection.name)) continue;
-
-      const isUsersCollection = this.userMappingService.isUsersCollection(collection.name);
-      
-      // Handle users collection first if needed
-      if (isUsersCollection && !this.hasImportedUsers) {
-        await this.importUsersCollection();
-      }
-
-      await this.importSingleCollection(db, collection);
-    }
-  }
-
-  /**
-   * Imports a single collection with batching and rate limiting.
-   */
-  private async importSingleCollection(
-    db: ConfigDatabase,
-    collection: CollectionCreate
-  ): Promise<void> {
-    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-    if (!collectionData || collectionData.data.length === 0) {
-      logger.info(`No data to import for collection: ${collection.name}`);
-      return;
-    }
-
-    logger.info(`Importing collection: ${collection.name} (${collectionData.data.length} items)`);
-
-    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-    const adapter = new LegacyAdapter(this.database.client);
-    if (operationId) {
-      await updateOperation(adapter, db.$id, operationId, { status: "in_progress" });
-    }
-
-    // Create batches for processing
-    const batches = this.createBatches(collectionData.data, this.batchLimit);
-    let processedItems = 0;
-
-    for (let i = 0; i < batches.length; i++) {
-      const batch = batches[i];
-      logger.info(`Processing batch ${i + 1} of ${batches.length} (${batch.length} items)`);
-
-      // Process batch with rate limiting
-      const batchPromises = batch.map((item, index) =>
-        this.rateLimitManager.dataInsertion(() => this.importSingleItem(db, collection, item))
-      );
-
-      const results = await Promise.allSettled(batchPromises);
-      
-      // Count successful imports
-      const successCount = results.filter(r => r.status === "fulfilled").length;
-      processedItems += successCount;
-
-      logger.info(`Batch ${i + 1} completed: ${successCount}/${batch.length} items imported`);
-
-      // Update operation progress
-      if (operationId) {
-        await updateOperation(adapter, db.$id, operationId, { progress: processedItems });
-      }
-    }
-
-    // Mark operation as completed
-    if (operationId) {
-      await updateOperation(adapter, db.$id, operationId, { status: "completed" });
-    }
-
-    logger.info(`Completed importing collection: ${collection.name} (${processedItems} items)`);
-  }
-
-  /**
-   * Imports a single item with error handling.
-   */
-  private async importSingleItem(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    item: any
-  ): Promise<void> {
-    try {
-      const id = item.finalData.docId || item.finalData.userId || item.context.docId || item.context.userId;
-
-      // Clean up internal fields
-      const cleanedData = { ...item.finalData };
-      delete cleanedData.userId;
-      delete cleanedData.docId;
-
-      if (!cleanedData || Object.keys(cleanedData).length === 0) {
-        return;
-      }
-
-      await tryAwaitWithRetry(
-        async () => await this.database.createDocument(db.$id, collection.$id!, id, cleanedData)
-      );
-
-    } catch (error) {
-      logger.error(`Error importing item to collection ${collection.name}:`, error);
-      throw error;
-    }
-  }
-
-  /**
-   * Helper method to generate consistent collection keys.
-   */
-  private getCollectionKey(name: string): string {
-    return name.toLowerCase().replace(" ", "");
-  }
-
-  /**
-   * Loads data from file based on import definition.
-   */
-  private loadDataFromFile(importDef: ImportDef): any[] {
-    try {
-      const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
-      
-      if (!fs.existsSync(filePath)) {
-        logger.error(`Import file not found: ${filePath}`);
-        return [];
-      }
-
-      const rawData = fs.readFileSync(filePath, "utf8");
-      const parsedData = importDef.basePath
-        ? JSON.parse(rawData)[importDef.basePath]
-        : JSON.parse(rawData);
-
-      logger.info(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
-      return parsedData || [];
-
-    } catch (error) {
-      logger.error(`Error loading data from file ${importDef.filePath}:`, error);
-      return [];
-    }
-  }
-
-  /**
-   * Creates batches for processing with the specified batch size.
-   */
-  private createBatches<T>(data: T[], batchSize: number): T[][] {
-    const batches: T[][] = [];
-    for (let i = 0; i < data.length; i += batchSize) {
-      batches.push(data.slice(i, i + batchSize));
-    }
-    return batches;
-  }
-
-  /**
-   * Generates a unique ID for documents.
-   */
-  private generateUniqueId(): string {
-    return ID.unique();
-  }
-
-  // Additional helper methods...
-  private async findExistingCollection(dbId: string, collection: CollectionCreate): Promise<any> {
-    // Implementation to find existing collection (preserve existing logic)
-    try {
-      const collections = await this.database.listCollections(dbId);
-      return collections.collections.find(c => c.name === collection.name || c.$id === collection.$id);
-    } catch (error) {
-      logger.error(`Error finding collection ${collection.name}:`, error);
-      return null;
-    }
-  }
-
-  private async updateOperationStatus(db: ConfigDatabase, collection: CollectionCreate, status: string, total?: number): Promise<void> {
-    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-    if (operationId) {
-      const updateData = total ? { status, total } : { status };
-      const adapter = new LegacyAdapter(this.database.client);
-      await updateOperation(adapter, db.$id, operationId, updateData);
-    }
-  }
-
-  private async importUsersCollection(): Promise<void> {
-    // Implementation for importing users collection (preserve existing logic)
-    // This would handle the sophisticated user import logic
-    this.hasImportedUsers = true;
-  }
-
-  private async prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): Promise<void> {
-    // Implementation for preparing update data (preserve existing logic)
-    // This would handle the update logic from the original DataLoader
-  }
-
-  private async executePostImportActions(dbId: string, specificCollections?: string[]): Promise<void> {
-    // Implementation for executing post-import actions (preserve existing logic)
-    // This would handle file uploads and other post-import actions
-  }
-
-  private async transferDataBetweenDatabases(sourceDb: Models.Database, targetDb: Models.Database): Promise<void> {
-    // Implementation for transferring data between databases (preserve existing logic)
-    // This would handle the existing transfer logic
-  }
+import {
+  ID,
+  Query,
+  type Databases,
+  type Models,
+  type Storage,
+} from "node-appwrite";
+import type {
+  AppwriteConfig,
+  ConfigCollection,
+  ConfigDatabase,
+  CollectionCreate,
+  ImportDef,
+} from "appwrite-utils";
+import path from "path";
+import fs from "fs";
+import { DataTransformationService } from "./DataTransformationService.js";
+import { RateLimitManager, type RateLimitConfig } from "./RateLimitManager.js";
+import { FileHandlerService } from "./FileHandlerService.js";
+import { UserMappingService } from "./UserMappingService.js";
+import { ValidationService } from "./ValidationService.js";
+import { RelationshipResolver, type CollectionImportData } from "./RelationshipResolver.js";
+import type { ImportDataActions } from "../importDataActions.js";
+import type { SetupOptions } from "../../utilsController.js";
+import { UsersController } from "../../users/methods.js";
+import { logger } from 'appwrite-utils-helpers';
+import { MessageFormatter } from 'appwrite-utils-helpers';
+import { ProgressManager } from "../../shared/progressManager.js";
+import { tryAwaitWithRetry } from "appwrite-utils-helpers";
+import { updateOperation, findOrCreateOperation } from "../../shared/migrationHelpers.js";
+import { AdapterFactory, type DatabaseAdapter } from 'appwrite-utils-helpers';
+import { resolveAndUpdateRelationships } from "../relationships.js";
+
+// Enhanced rate limiting configuration - now managed by RateLimitManager
+
+/**
+ * Orchestrator for the entire import process.
+ * Coordinates all services while preserving existing functionality and performance characteristics.
+ * 
+ * This replaces the monolithic ImportController and DataLoader with a cleaner, modular architecture.
+ */
+export class ImportOrchestrator {
+  // Core dependencies
+  private config: AppwriteConfig;
+  private database: Databases;
+  private storage: Storage;
+  private appwriteFolderPath: string;
+  private setupOptions: SetupOptions;
+  private databasesToRun: Models.Database[];
+
+  // Services
+  private dataTransformationService: DataTransformationService;
+  private fileHandlerService: FileHandlerService;
+  private userMappingService: UserMappingService;
+  private validationService: ValidationService;
+  private relationshipResolver: RelationshipResolver;
+  private rateLimitManager: RateLimitManager;
+
+  // Import state
+  private importMap = new Map<string, CollectionImportData>();
+  private collectionImportOperations = new Map<string, string>();
+  private hasImportedUsers = false;
+  private batchLimit: number = 50; // Preserve existing batch size
+  private _adapter: DatabaseAdapter | null = null;
+
+  private async getAdapter(): Promise<DatabaseAdapter> {
+    if (!this._adapter) {
+      const { adapter } = await AdapterFactory.createFromConfig(this.config);
+      this._adapter = adapter;
+    }
+    return this._adapter;
+  }
+
+  constructor(
+    config: AppwriteConfig,
+    database: Databases,
+    storage: Storage,
+    appwriteFolderPath: string,
+    importDataActions: ImportDataActions,
+    setupOptions: SetupOptions,
+    databasesToRun?: Models.Database[],
+    rateLimitConfig?: Partial<RateLimitConfig>
+  ) {
+    this.config = config;
+    this.database = database;
+    this.storage = storage;
+    this.appwriteFolderPath = appwriteFolderPath;
+    this.setupOptions = setupOptions;
+    this.databasesToRun = databasesToRun || [];
+
+    // Initialize services
+    this.rateLimitManager = new RateLimitManager(rateLimitConfig);
+    this.dataTransformationService = new DataTransformationService(importDataActions);
+    this.fileHandlerService = new FileHandlerService(appwriteFolderPath, config, importDataActions, this.rateLimitManager);
+    this.userMappingService = new UserMappingService(config, this.dataTransformationService);
+    this.validationService = new ValidationService(importDataActions);
+    this.relationshipResolver = new RelationshipResolver(config, this.userMappingService);
+  }
+
+  /**
+   * Main entry point for the import process.
+   * Preserves existing import flow while using the new modular architecture.
+   */
+  async run(specificCollections?: string[]): Promise<void> {
+    let databasesToProcess: Models.Database[];
+
+    if (this.databasesToRun.length > 0) {
+      databasesToProcess = this.databasesToRun;
+    } else {
+      const allDatabases = await this.database.list();
+      databasesToProcess = allDatabases.databases;
+    }
+
+    let processedDatabase: Models.Database | undefined;
+
+    for (const db of databasesToProcess) {
+      MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
+
+      if (!processedDatabase) {
+        processedDatabase = db;
+        await this.performDatabaseImport(db, specificCollections);
+      } else if (processedDatabase.$id !== db.$id) {
+        await this.transferDataBetweenDatabases(processedDatabase, db);
+      }
+
+      console.log(`---------------------------------`);
+      console.log(`Finished import data for database: ${db.name}`);
+      console.log(`---------------------------------`);
+    }
+  }
+
+  /**
+   * Performs the complete import process for a single database.
+   */
+  private async performDatabaseImport(
+    db: Models.Database,
+    specificCollections?: string[]
+  ): Promise<void> {
+    try {
+      // Step 1: Setup and validation
+      await this.setupImportMaps(db.$id);
+      await this.loadExistingUsers();
+      
+      // Step 2: Pre-import validation
+      const validationResult = this.validationService.performPreImportValidation(
+        this.config.collections || [],
+        this.appwriteFolderPath
+      );
+      
+      if (!validationResult.isValid) {
+        logger.error("Pre-import validation failed:");
+        validationResult.errors.forEach(error => logger.error(`  - ${error}`));
+        throw new Error("Import validation failed");
+      }
+
+      if (validationResult.warnings.length > 0) {
+        logger.warn("Pre-import validation warnings:");
+        validationResult.warnings.forEach(warning => logger.warn(`  - ${warning}`));
+      }
+
+      // Step 3: Load and prepare data
+      await this.loadAndPrepareData(db, specificCollections);
+
+      // Step 4: Resolve relationships
+      logger.info("Resolving relationships...");
+      this.relationshipResolver.updateOldReferencesForNew(
+        this.importMap,
+        this.config.collections || []
+      );
+
+      // Step 5: Import collections
+      await this.importCollections(db, specificCollections);
+
+      // Step 6: Resolve and update relationships (existing logic)
+      await resolveAndUpdateRelationships(db.$id, this.database, this.config);
+
+      // Step 7: Execute post-import actions
+      await this.executePostImportActions(db.$id, specificCollections);
+
+    } catch (error) {
+      logger.error(`Error during database import for ${db.name}:`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Sets up import maps and operation tracking.
+   * Preserves existing setup logic from DataLoader.
+   */
+  private async setupImportMaps(dbId: string): Promise<void> {
+    // Initialize the users collection in the import map
+    this.importMap.set(this.getCollectionKey("users"), { data: [] });
+
+    for (const db of this.config.databases) {
+      if (db.$id !== dbId) continue;
+      if (!this.config.collections) continue;
+
+      for (let index = 0; index < this.config.collections.length; index++) {
+        const collectionConfig = this.config.collections[index];
+        const collection = { ...collectionConfig } as CollectionCreate;
+
+        // Check if the collection exists in the database (existing logic)
+        const existingCollection = await this.findExistingCollection(db.$id, collection);
+        if (!existingCollection) {
+          logger.error(`No collection found for ${collection.name}`);
+          continue;
+        }
+
+        // Update the collection ID with the existing one
+        collectionConfig.$id = existingCollection.$id;
+        collection.$id = existingCollection.$id;
+        this.config.collections[index] = collectionConfig;
+
+        // Find or create an import operation for the collection
+        const adapter = await this.getAdapter();
+        const collectionImportOperation = await findOrCreateOperation(
+          adapter,
+          dbId,
+          "importData",
+          collection.$id!
+        );
+        this.collectionImportOperations.set(
+          this.getCollectionKey(collection.name),
+          collectionImportOperation.$id
+        );
+
+        // Initialize the collection in the import map
+        this.importMap.set(this.getCollectionKey(collection.name), {
+          collection: collection,
+          data: [],
+        });
+      }
+    }
+  }
+
+  /**
+   * Loads existing users and initializes user mapping service.
+   */
+  private async loadExistingUsers(): Promise<void> {
+    const users = new UsersController(this.config, this.database);
+    const allUsers = await users.getAllUsers();
+    
+    // Initialize user mapping service with existing users
+    this.userMappingService.initializeWithExistingUsers(allUsers);
+
+    // Add existing users to import map (preserve existing logic)
+    const usersImportData = this.importMap.get(this.getCollectionKey("users"));
+    if (usersImportData) {
+      for (const user of allUsers) {
+        usersImportData.data.push({
+          finalData: {
+            ...user,
+            email: user.email?.toLowerCase(),
+            userId: user.$id,
+            docId: user.$id,
+          },
+          context: {
+            ...user,
+            email: user.email?.toLowerCase(),
+            userId: user.$id,
+            docId: user.$id,
+          },
+          rawData: user,
+        });
+      }
+      this.importMap.set(this.getCollectionKey("users"), usersImportData);
+    }
+
+    logger.info(`Loaded ${allUsers.length} existing users for deduplication`);
+  }
+
+  /**
+   * Loads and prepares data for all collections.
+   */
+  private async loadAndPrepareData(
+    db: ConfigDatabase,
+    specificCollections?: string[]
+  ): Promise<void> {
+    const collectionsToProcess = specificCollections || 
+      (this.config.collections ? this.config.collections.map(c => c.name) : []);
+
+    for (const collectionConfig of this.config.collections || []) {
+      if (!collectionsToProcess.includes(collectionConfig.name)) continue;
+      if (!collectionConfig.importDefs || collectionConfig.importDefs.length === 0) continue;
+
+      const isUsersCollection = this.userMappingService.isUsersCollection(collectionConfig.name);
+
+      // Process create definitions
+      const createDefs = collectionConfig.importDefs.filter(
+        (def: ImportDef) => def.type === "create" || !def.type
+      );
+
+      for (const createDef of createDefs) {
+        if (isUsersCollection && createDef.createUsers) {
+          await this.prepareUserCollectionData(db, collectionConfig, createDef);
+        } else {
+          await this.prepareCollectionData(db, collectionConfig, createDef);
+        }
+      }
+
+      // Process update definitions
+      const updateDefs = collectionConfig.importDefs.filter(
+        (def: ImportDef) => def.type === "update"
+      );
+
+      for (const updateDef of updateDefs) {
+        await this.prepareUpdateData(db, collectionConfig, updateDef);
+      }
+    }
+  }
+
+  /**
+   * Prepares data for a regular collection.
+   * Uses the DataTransformationService for all transformations.
+   */
+  private async prepareCollectionData(
+    db: ConfigDatabase,
+    collection: CollectionCreate,
+    importDef: ImportDef
+  ): Promise<void> {
+    const rawData = this.loadDataFromFile(importDef);
+    if (rawData.length === 0) return;
+
+    await this.updateOperationStatus(db, collection, "ready", rawData.length);
+
+    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
+    if (!collectionData) {
+      logger.error(`No collection data found for ${collection.name}`);
+      return;
+    }
+
+    for (const item of rawData) {
+      try {
+        // Generate unique ID
+        const itemId = this.generateUniqueId();
+        
+        // Create context
+        const context = this.dataTransformationService.createContext(db, collection, item, itemId);
+        
+        // Transform data
+        const transformedData = this.dataTransformationService.transformData(
+          item,
+          importDef.attributeMappings
+        );
+
+        // Validate transformed data
+        const isValid = this.dataTransformationService.validateTransformedData(
+          transformedData,
+          importDef.attributeMappings,
+          context
+        );
+
+        if (!isValid) {
+          logger.warn(`Skipping invalid item: ${JSON.stringify(item, null, 2)}`);
+          continue;
+        }
+
+        // Handle file mappings
+        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
+          importDef.attributeMappings,
+          context,
+          transformedData
+        );
+
+        // Store ID mapping if primary key exists
+        if (importDef.primaryKeyField) {
+          const oldId = item[importDef.primaryKeyField];
+          if (this.relationshipResolver.hasIdMapping(collection.name, oldId)) {
+            logger.error(`Duplicate primary key ${oldId} in collection ${collection.name}`);
+            continue;
+          }
+          this.relationshipResolver.setIdMapping(collection.name, oldId, itemId);
+        }
+
+        // Add to collection data
+        collectionData.data.push({
+          rawData: item,
+          context: { ...context, ...transformedData },
+          importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
+          finalData: transformedData,
+        });
+
+      } catch (error) {
+        logger.error(`Error preparing item for collection ${collection.name}:`, error);
+        continue;
+      }
+    }
+
+    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
+  }
+
+  /**
+   * Prepares data for user collection with deduplication.
+   * Uses the UserMappingService for sophisticated user handling.
+   */
+  private async prepareUserCollectionData(
+    db: ConfigDatabase,
+    collection: CollectionCreate,
+    importDef: ImportDef
+  ): Promise<void> {
+    const rawData = this.loadDataFromFile(importDef);
+    if (rawData.length === 0) return;
+
+    await this.updateOperationStatus(db, collection, "ready", rawData.length);
+
+    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
+    if (!collectionData) return;
+
+    for (const item of rawData) {
+      try {
+        const proposedId = this.userMappingService.getTrueUniqueUserId(collection.name);
+        
+        // Prepare user data with deduplication
+        const { transformedItem, existingId, userData } = this.userMappingService.prepareUserData(
+          item,
+          importDef.attributeMappings,
+          importDef.primaryKeyField,
+          proposedId
+        );
+
+        const finalId = existingId || proposedId;
+        const context = this.dataTransformationService.createContext(db, collection, item, finalId);
+
+        // Handle file mappings
+        const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(
+          importDef.attributeMappings,
+          context,
+          transformedItem
+        );
+
+        // Store ID mapping
+        if (importDef.primaryKeyField) {
+          const oldId = item[importDef.primaryKeyField];
+          this.relationshipResolver.setIdMapping(collection.name, oldId, finalId);
+        }
+
+        // Check for existing data and merge if needed
+        const existingDataIndex = collectionData.data.findIndex(data =>
+          data.finalData.docId === finalId || data.finalData.userId === finalId
+        );
+
+        if (existingDataIndex >= 0) {
+          // Merge with existing data
+          const existingData = collectionData.data[existingDataIndex];
+          existingData.finalData = this.dataTransformationService.mergeObjects(
+            existingData.finalData,
+            transformedItem
+          );
+          existingData.context = this.dataTransformationService.mergeObjects(
+            existingData.context,
+            { ...context, ...transformedItem, ...userData.finalData }
+          );
+        } else {
+          // Add new data
+          collectionData.data.push({
+            rawData: item,
+            context: { ...context, ...transformedItem, ...userData.finalData },
+            importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
+            finalData: transformedItem,
+          });
+        }
+
+      } catch (error) {
+        logger.error(`Error preparing user data for collection ${collection.name}:`, error);
+        continue;
+      }
+    }
+
+    this.importMap.set(this.getCollectionKey(collection.name), collectionData);
+  }
+
+  /**
+   * Imports collections with rate limiting and batch processing.
+   * Preserves existing import logic with enhanced error handling.
+   */
+  private async importCollections(
+    db: ConfigDatabase,
+    specificCollections?: string[]
+  ): Promise<void> {
+    const collectionsToImport = specificCollections ||
+      (this.config.collections ? this.config.collections.map(c => c.name) : []);
+
+    for (const collection of this.config.collections || []) {
+      if (!collectionsToImport.includes(collection.name)) continue;
+
+      const isUsersCollection = this.userMappingService.isUsersCollection(collection.name);
+      
+      // Handle users collection first if needed
+      if (isUsersCollection && !this.hasImportedUsers) {
+        await this.importUsersCollection();
+      }
+
+      await this.importSingleCollection(db, collection);
+    }
+  }
+
+  /**
+   * Imports a single collection with batching and rate limiting.
+   */
+  private async importSingleCollection(
+    db: ConfigDatabase,
+    collection: CollectionCreate
+  ): Promise<void> {
+    const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
+    if (!collectionData || collectionData.data.length === 0) {
+      logger.info(`No data to import for collection: ${collection.name}`);
+      return;
+    }
+
+    logger.info(`Importing collection: ${collection.name} (${collectionData.data.length} items)`);
+
+    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
+    const adapter = await this.getAdapter();
+    if (operationId) {
+      await updateOperation(adapter, db.$id, operationId, { status: "in_progress" });
+    }
+
+    // Create batches for processing
+    const batches = this.createBatches(collectionData.data, this.batchLimit);
+    let processedItems = 0;
+
+    for (let i = 0; i < batches.length; i++) {
+      const batch = batches[i];
+      logger.info(`Processing batch ${i + 1} of ${batches.length} (${batch.length} items)`);
+
+      // Process batch with rate limiting
+      const batchPromises = batch.map((item, index) =>
+        this.rateLimitManager.dataInsertion(() => this.importSingleItem(db, collection, item))
+      );
+
+      const results = await Promise.allSettled(batchPromises);
+      
+      // Count successful imports
+      const successCount = results.filter(r => r.status === "fulfilled").length;
+      processedItems += successCount;
+
+      logger.info(`Batch ${i + 1} completed: ${successCount}/${batch.length} items imported`);
+
+      // Update operation progress
+      if (operationId) {
+        await updateOperation(adapter, db.$id, operationId, { progress: processedItems });
+      }
+    }
+
+    // Mark operation as completed
+    if (operationId) {
+      await updateOperation(adapter, db.$id, operationId, { status: "completed" });
+    }
+
+    logger.info(`Completed importing collection: ${collection.name} (${processedItems} items)`);
+  }
+
+  /**
+   * Imports a single item with error handling.
+   */
+  private async importSingleItem(
+    db: ConfigDatabase,
+    collection: CollectionCreate,
+    item: any
+  ): Promise<void> {
+    try {
+      const id = item.finalData.docId || item.finalData.userId || item.context.docId || item.context.userId;
+
+      // Clean up internal fields
+      const cleanedData = { ...item.finalData };
+      delete cleanedData.userId;
+      delete cleanedData.docId;
+
+      if (!cleanedData || Object.keys(cleanedData).length === 0) {
+        return;
+      }
+
+      await tryAwaitWithRetry(
+        async () => await this.database.createDocument(db.$id, collection.$id!, id, cleanedData)
+      );
+
+    } catch (error) {
+      logger.error(`Error importing item to collection ${collection.name}:`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Helper method to generate consistent collection keys.
+   */
+  private getCollectionKey(name: string): string {
+    return name.toLowerCase().replace(" ", "");
+  }
+
+  /**
+   * Loads data from file based on import definition.
+   */
+  private loadDataFromFile(importDef: ImportDef): any[] {
+    try {
+      const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
+      
+      if (!fs.existsSync(filePath)) {
+        logger.error(`Import file not found: ${filePath}`);
+        return [];
+      }
+
+      const rawData = fs.readFileSync(filePath, "utf8");
+      const parsedData = importDef.basePath
+        ? JSON.parse(rawData)[importDef.basePath]
+        : JSON.parse(rawData);
+
+      logger.info(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
+      return parsedData || [];
+
+    } catch (error) {
+      logger.error(`Error loading data from file ${importDef.filePath}:`, error);
+      return [];
+    }
+  }
+
+  /**
+   * Creates batches for processing with the specified batch size.
+   */
+  private createBatches<T>(data: T[], batchSize: number): T[][] {
+    const batches: T[][] = [];
+    for (let i = 0; i < data.length; i += batchSize) {
+      batches.push(data.slice(i, i + batchSize));
+    }
+    return batches;
+  }
+
+  /**
+   * Generates a unique ID for documents.
+   */
+  private generateUniqueId(): string {
+    return ID.unique();
+  }
+
+  // Additional helper methods...
+  private async findExistingCollection(dbId: string, collection: CollectionCreate): Promise<any> {
+    // Implementation to find existing collection (preserve existing logic)
+    try {
+      const collections = await this.database.listCollections(dbId);
+      return collections.collections.find(c => c.name === collection.name || c.$id === collection.$id);
+    } catch (error) {
+      logger.error(`Error finding collection ${collection.name}:`, error);
+      return null;
+    }
+  }
+
+  private async updateOperationStatus(db: ConfigDatabase, collection: CollectionCreate, status: string, total?: number): Promise<void> {
+    const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
+    if (operationId) {
+      const updateData = total ? { status, total } : { status };
+      const adapter = await this.getAdapter();
+      await updateOperation(adapter, db.$id, operationId, updateData);
+    }
+  }
+
+  private async importUsersCollection(): Promise<void> {
+    // Implementation for importing users collection (preserve existing logic)
+    // This would handle the sophisticated user import logic
+    this.hasImportedUsers = true;
+  }
+
+  private async prepareUpdateData(db: ConfigDatabase, collection: CollectionCreate, importDef: ImportDef): Promise<void> {
+    // Implementation for preparing update data (preserve existing logic)
+    // This would handle the update logic from the original DataLoader
+  }
+
+  private async executePostImportActions(dbId: string, specificCollections?: string[]): Promise<void> {
+    // Implementation for executing post-import actions (preserve existing logic)
+    // This would handle file uploads and other post-import actions
+  }
+
+  private async transferDataBetweenDatabases(sourceDb: Models.Database, targetDb: Models.Database): Promise<void> {
+    // Implementation for transferring data between databases (preserve existing logic)
+    // This would handle the existing transfer logic
+  }
 }