appwrite-utils-cli 1.9.6 → 1.11.0

package/dist/migrations/services/ImportOrchestrator.js

@@ -1,485 +0,0 @@
-import { ID, Query, } from "node-appwrite";
-import path from "path";
-import fs from "fs";
-import { DataTransformationService } from "./DataTransformationService.js";
-import { RateLimitManager } from "./RateLimitManager.js";
-import { FileHandlerService } from "./FileHandlerService.js";
-import { UserMappingService } from "./UserMappingService.js";
-import { ValidationService } from "./ValidationService.js";
-import { RelationshipResolver } from "./RelationshipResolver.js";
-import { UsersController } from "../../users/methods.js";
-import { logger } from "../../shared/logging.js";
-import { MessageFormatter } from "../../shared/messageFormatter.js";
-import { ProgressManager } from "../../shared/progressManager.js";
-import { tryAwaitWithRetry } from "../../utils/index.js";
-import { updateOperation, findOrCreateOperation } from "../../shared/migrationHelpers.js";
-import { LegacyAdapter } from "../../adapters/LegacyAdapter.js";
-import { resolveAndUpdateRelationships } from "../relationships.js";
-// Enhanced rate limiting configuration - now managed by RateLimitManager
-/**
- * Orchestrator for the entire import process.
- * Coordinates all services while preserving existing functionality and performance characteristics.
- *
- * This replaces the monolithic ImportController and DataLoader with a cleaner, modular architecture.
- */
-export class ImportOrchestrator {
-    // Core dependencies
-    config;
-    database;
-    storage;
-    appwriteFolderPath;
-    setupOptions;
-    databasesToRun;
-    // Services
-    dataTransformationService;
-    fileHandlerService;
-    userMappingService;
-    validationService;
-    relationshipResolver;
-    rateLimitManager;
-    // Import state
-    importMap = new Map();
-    collectionImportOperations = new Map();
-    hasImportedUsers = false;
-    batchLimit = 50; // Preserve existing batch size
-    constructor(config, database, storage, appwriteFolderPath, importDataActions, setupOptions, databasesToRun, rateLimitConfig) {
-        this.config = config;
-        this.database = database;
-        this.storage = storage;
-        this.appwriteFolderPath = appwriteFolderPath;
-        this.setupOptions = setupOptions;
-        this.databasesToRun = databasesToRun || [];
-        // Initialize services
-        this.rateLimitManager = new RateLimitManager(rateLimitConfig);
-        this.dataTransformationService = new DataTransformationService(importDataActions);
-        this.fileHandlerService = new FileHandlerService(appwriteFolderPath, config, importDataActions, this.rateLimitManager);
-        this.userMappingService = new UserMappingService(config, this.dataTransformationService);
-        this.validationService = new ValidationService(importDataActions);
-        this.relationshipResolver = new RelationshipResolver(config, this.userMappingService);
-    }
-    /**
-     * Main entry point for the import process.
-     * Preserves existing import flow while using the new modular architecture.
-     */
-    async run(specificCollections) {
-        let databasesToProcess;
-        if (this.databasesToRun.length > 0) {
-            databasesToProcess = this.databasesToRun;
-        }
-        else {
-            const allDatabases = await this.database.list();
-            databasesToProcess = allDatabases.databases;
-        }
-        let processedDatabase;
-        for (const db of databasesToProcess) {
-            MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
-            if (!processedDatabase) {
-                processedDatabase = db;
-                await this.performDatabaseImport(db, specificCollections);
-            }
-            else if (processedDatabase.$id !== db.$id) {
-                await this.transferDataBetweenDatabases(processedDatabase, db);
-            }
-            console.log(`---------------------------------`);
-            console.log(`Finished import data for database: ${db.name}`);
-            console.log(`---------------------------------`);
-        }
-    }
-    /**
-     * Performs the complete import process for a single database.
-     */
-    async performDatabaseImport(db, specificCollections) {
-        try {
-            // Step 1: Setup and validation
-            await this.setupImportMaps(db.$id);
-            await this.loadExistingUsers();
-            // Step 2: Pre-import validation
-            const validationResult = this.validationService.performPreImportValidation(this.config.collections || [], this.appwriteFolderPath);
-            if (!validationResult.isValid) {
-                logger.error("Pre-import validation failed:");
-                validationResult.errors.forEach(error => logger.error(`  - ${error}`));
-                throw new Error("Import validation failed");
-            }
-            if (validationResult.warnings.length > 0) {
-                logger.warn("Pre-import validation warnings:");
-                validationResult.warnings.forEach(warning => logger.warn(`  - ${warning}`));
-            }
-            // Step 3: Load and prepare data
-            await this.loadAndPrepareData(db, specificCollections);
-            // Step 4: Resolve relationships
-            logger.info("Resolving relationships...");
-            this.relationshipResolver.updateOldReferencesForNew(this.importMap, this.config.collections || []);
-            // Step 5: Import collections
-            await this.importCollections(db, specificCollections);
-            // Step 6: Resolve and update relationships (existing logic)
-            await resolveAndUpdateRelationships(db.$id, this.database, this.config);
-            // Step 7: Execute post-import actions
-            await this.executePostImportActions(db.$id, specificCollections);
-        }
-        catch (error) {
-            logger.error(`Error during database import for ${db.name}:`, error);
-            throw error;
-        }
-    }
-    /**
-     * Sets up import maps and operation tracking.
-     * Preserves existing setup logic from DataLoader.
-     */
-    async setupImportMaps(dbId) {
-        // Initialize the users collection in the import map
-        this.importMap.set(this.getCollectionKey("users"), { data: [] });
-        for (const db of this.config.databases) {
-            if (db.$id !== dbId)
-                continue;
-            if (!this.config.collections)
-                continue;
-            for (let index = 0; index < this.config.collections.length; index++) {
-                const collectionConfig = this.config.collections[index];
-                const collection = { ...collectionConfig };
-                // Check if the collection exists in the database (existing logic)
-                const existingCollection = await this.findExistingCollection(db.$id, collection);
-                if (!existingCollection) {
-                    logger.error(`No collection found for ${collection.name}`);
-                    continue;
-                }
-                // Update the collection ID with the existing one
-                collectionConfig.$id = existingCollection.$id;
-                collection.$id = existingCollection.$id;
-                this.config.collections[index] = collectionConfig;
-                // Find or create an import operation for the collection
-                const adapter = new LegacyAdapter(this.database.client);
-                const collectionImportOperation = await findOrCreateOperation(adapter, dbId, "importData", collection.$id);
-                this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
-                // Initialize the collection in the import map
-                this.importMap.set(this.getCollectionKey(collection.name), {
-                    collection: collection,
-                    data: [],
-                });
-            }
-        }
-    }
-    /**
-     * Loads existing users and initializes user mapping service.
-     */
-    async loadExistingUsers() {
-        const users = new UsersController(this.config, this.database);
-        const allUsers = await users.getAllUsers();
-        // Initialize user mapping service with existing users
-        this.userMappingService.initializeWithExistingUsers(allUsers);
-        // Add existing users to import map (preserve existing logic)
-        const usersImportData = this.importMap.get(this.getCollectionKey("users"));
-        if (usersImportData) {
-            for (const user of allUsers) {
-                usersImportData.data.push({
-                    finalData: {
-                        ...user,
-                        email: user.email?.toLowerCase(),
-                        userId: user.$id,
-                        docId: user.$id,
-                    },
-                    context: {
-                        ...user,
-                        email: user.email?.toLowerCase(),
-                        userId: user.$id,
-                        docId: user.$id,
-                    },
-                    rawData: user,
-                });
-            }
-            this.importMap.set(this.getCollectionKey("users"), usersImportData);
-        }
-        logger.info(`Loaded ${allUsers.length} existing users for deduplication`);
-    }
-    /**
-     * Loads and prepares data for all collections.
-     */
-    async loadAndPrepareData(db, specificCollections) {
-        const collectionsToProcess = specificCollections ||
-            (this.config.collections ? this.config.collections.map(c => c.name) : []);
-        for (const collectionConfig of this.config.collections || []) {
-            if (!collectionsToProcess.includes(collectionConfig.name))
-                continue;
-            if (!collectionConfig.importDefs || collectionConfig.importDefs.length === 0)
-                continue;
-            const isUsersCollection = this.userMappingService.isUsersCollection(collectionConfig.name);
-            // Process create definitions
-            const createDefs = collectionConfig.importDefs.filter((def) => def.type === "create" || !def.type);
-            for (const createDef of createDefs) {
-                if (isUsersCollection && createDef.createUsers) {
-                    await this.prepareUserCollectionData(db, collectionConfig, createDef);
-                }
-                else {
-                    await this.prepareCollectionData(db, collectionConfig, createDef);
-                }
-            }
-            // Process update definitions
-            const updateDefs = collectionConfig.importDefs.filter((def) => def.type === "update");
-            for (const updateDef of updateDefs) {
-                await this.prepareUpdateData(db, collectionConfig, updateDef);
-            }
-        }
-    }
-    /**
-     * Prepares data for a regular collection.
-     * Uses the DataTransformationService for all transformations.
-     */
-    async prepareCollectionData(db, collection, importDef) {
-        const rawData = this.loadDataFromFile(importDef);
-        if (rawData.length === 0)
-            return;
-        await this.updateOperationStatus(db, collection, "ready", rawData.length);
-        const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-        if (!collectionData) {
-            logger.error(`No collection data found for ${collection.name}`);
-            return;
-        }
-        for (const item of rawData) {
-            try {
-                // Generate unique ID
-                const itemId = this.generateUniqueId();
-                // Create context
-                const context = this.dataTransformationService.createContext(db, collection, item, itemId);
-                // Transform data
-                const transformedData = this.dataTransformationService.transformData(item, importDef.attributeMappings);
-                // Validate transformed data
-                const isValid = this.dataTransformationService.validateTransformedData(transformedData, importDef.attributeMappings, context);
-                if (!isValid) {
-                    logger.warn(`Skipping invalid item: ${JSON.stringify(item, null, 2)}`);
-                    continue;
-                }
-                // Handle file mappings
-                const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(importDef.attributeMappings, context, transformedData);
-                // Store ID mapping if primary key exists
-                if (importDef.primaryKeyField) {
-                    const oldId = item[importDef.primaryKeyField];
-                    if (this.relationshipResolver.hasIdMapping(collection.name, oldId)) {
-                        logger.error(`Duplicate primary key ${oldId} in collection ${collection.name}`);
-                        continue;
-                    }
-                    this.relationshipResolver.setIdMapping(collection.name, oldId, itemId);
-                }
-                // Add to collection data
-                collectionData.data.push({
-                    rawData: item,
-                    context: { ...context, ...transformedData },
-                    importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-                    finalData: transformedData,
-                });
-            }
-            catch (error) {
-                logger.error(`Error preparing item for collection ${collection.name}:`, error);
-                continue;
-            }
-        }
-        this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-    }
-    /**
-     * Prepares data for user collection with deduplication.
-     * Uses the UserMappingService for sophisticated user handling.
-     */
-    async prepareUserCollectionData(db, collection, importDef) {
-        const rawData = this.loadDataFromFile(importDef);
-        if (rawData.length === 0)
-            return;
-        await this.updateOperationStatus(db, collection, "ready", rawData.length);
-        const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-        if (!collectionData)
-            return;
-        for (const item of rawData) {
-            try {
-                const proposedId = this.userMappingService.getTrueUniqueUserId(collection.name);
-                // Prepare user data with deduplication
-                const { transformedItem, existingId, userData } = this.userMappingService.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, proposedId);
-                const finalId = existingId || proposedId;
-                const context = this.dataTransformationService.createContext(db, collection, item, finalId);
-                // Handle file mappings
-                const mappingsWithFileActions = this.fileHandlerService.getAttributeMappingsWithFileActions(importDef.attributeMappings, context, transformedItem);
-                // Store ID mapping
-                if (importDef.primaryKeyField) {
-                    const oldId = item[importDef.primaryKeyField];
-                    this.relationshipResolver.setIdMapping(collection.name, oldId, finalId);
-                }
-                // Check for existing data and merge if needed
-                const existingDataIndex = collectionData.data.findIndex(data => data.finalData.docId === finalId || data.finalData.userId === finalId);
-                if (existingDataIndex >= 0) {
-                    // Merge with existing data
-                    const existingData = collectionData.data[existingDataIndex];
-                    existingData.finalData = this.dataTransformationService.mergeObjects(existingData.finalData, transformedItem);
-                    existingData.context = this.dataTransformationService.mergeObjects(existingData.context, { ...context, ...transformedItem, ...userData.finalData });
-                }
-                else {
-                    // Add new data
-                    collectionData.data.push({
-                        rawData: item,
-                        context: { ...context, ...transformedItem, ...userData.finalData },
-                        importDef: { ...importDef, attributeMappings: mappingsWithFileActions },
-                        finalData: transformedItem,
-                    });
-                }
-            }
-            catch (error) {
-                logger.error(`Error preparing user data for collection ${collection.name}:`, error);
-                continue;
-            }
-        }
-        this.importMap.set(this.getCollectionKey(collection.name), collectionData);
-    }
-    /**
-     * Imports collections with rate limiting and batch processing.
-     * Preserves existing import logic with enhanced error handling.
-     */
-    async importCollections(db, specificCollections) {
-        const collectionsToImport = specificCollections ||
-            (this.config.collections ? this.config.collections.map(c => c.name) : []);
-        for (const collection of this.config.collections || []) {
-            if (!collectionsToImport.includes(collection.name))
-                continue;
-            const isUsersCollection = this.userMappingService.isUsersCollection(collection.name);
-            // Handle users collection first if needed
-            if (isUsersCollection && !this.hasImportedUsers) {
-                await this.importUsersCollection();
-            }
-            await this.importSingleCollection(db, collection);
-        }
-    }
-    /**
-     * Imports a single collection with batching and rate limiting.
-     */
-    async importSingleCollection(db, collection) {
-        const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-        if (!collectionData || collectionData.data.length === 0) {
-            logger.info(`No data to import for collection: ${collection.name}`);
-            return;
-        }
-        logger.info(`Importing collection: ${collection.name} (${collectionData.data.length} items)`);
-        const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-        const adapter = new LegacyAdapter(this.database.client);
-        if (operationId) {
-            await updateOperation(adapter, db.$id, operationId, { status: "in_progress" });
-        }
-        // Create batches for processing
-        const batches = this.createBatches(collectionData.data, this.batchLimit);
-        let processedItems = 0;
-        for (let i = 0; i < batches.length; i++) {
-            const batch = batches[i];
-            logger.info(`Processing batch ${i + 1} of ${batches.length} (${batch.length} items)`);
-            // Process batch with rate limiting
-            const batchPromises = batch.map((item, index) => this.rateLimitManager.dataInsertion(() => this.importSingleItem(db, collection, item)));
-            const results = await Promise.allSettled(batchPromises);
-            // Count successful imports
-            const successCount = results.filter(r => r.status === "fulfilled").length;
-            processedItems += successCount;
-            logger.info(`Batch ${i + 1} completed: ${successCount}/${batch.length} items imported`);
-            // Update operation progress
-            if (operationId) {
-                await updateOperation(adapter, db.$id, operationId, { progress: processedItems });
-            }
-        }
-        // Mark operation as completed
-        if (operationId) {
-            await updateOperation(adapter, db.$id, operationId, { status: "completed" });
-        }
-        logger.info(`Completed importing collection: ${collection.name} (${processedItems} items)`);
-    }
-    /**
-     * Imports a single item with error handling.
-     */
-    async importSingleItem(db, collection, item) {
-        try {
-            const id = item.finalData.docId || item.finalData.userId || item.context.docId || item.context.userId;
-            // Clean up internal fields
-            const cleanedData = { ...item.finalData };
-            delete cleanedData.userId;
-            delete cleanedData.docId;
-            if (!cleanedData || Object.keys(cleanedData).length === 0) {
-                return;
-            }
-            await tryAwaitWithRetry(async () => await this.database.createDocument(db.$id, collection.$id, id, cleanedData));
-        }
-        catch (error) {
-            logger.error(`Error importing item to collection ${collection.name}:`, error);
-            throw error;
-        }
-    }
-    /**
-     * Helper method to generate consistent collection keys.
-     */
-    getCollectionKey(name) {
-        return name.toLowerCase().replace(" ", "");
-    }
-    /**
-     * Loads data from file based on import definition.
-     */
-    loadDataFromFile(importDef) {
-        try {
-            const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
-            if (!fs.existsSync(filePath)) {
-                logger.error(`Import file not found: ${filePath}`);
-                return [];
-            }
-            const rawData = fs.readFileSync(filePath, "utf8");
-            const parsedData = importDef.basePath
-                ? JSON.parse(rawData)[importDef.basePath]
-                : JSON.parse(rawData);
-            logger.info(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
-            return parsedData || [];
-        }
-        catch (error) {
-            logger.error(`Error loading data from file ${importDef.filePath}:`, error);
-            return [];
-        }
-    }
-    /**
-     * Creates batches for processing with the specified batch size.
-     */
-    createBatches(data, batchSize) {
-        const batches = [];
-        for (let i = 0; i < data.length; i += batchSize) {
-            batches.push(data.slice(i, i + batchSize));
-        }
-        return batches;
-    }
-    /**
-     * Generates a unique ID for documents.
-     */
-    generateUniqueId() {
-        return ID.unique();
-    }
-    // Additional helper methods...
-    async findExistingCollection(dbId, collection) {
-        // Implementation to find existing collection (preserve existing logic)
-        try {
-            const collections = await this.database.listCollections(dbId);
-            return collections.collections.find(c => c.name === collection.name || c.$id === collection.$id);
-        }
-        catch (error) {
-            logger.error(`Error finding collection ${collection.name}:`, error);
-            return null;
-        }
-    }
-    async updateOperationStatus(db, collection, status, total) {
-        const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
-        if (operationId) {
-            const updateData = total ? { status, total } : { status };
-            const adapter = new LegacyAdapter(this.database.client);
-            await updateOperation(adapter, db.$id, operationId, updateData);
-        }
-    }
-    async importUsersCollection() {
-        // Implementation for importing users collection (preserve existing logic)
-        // This would handle the sophisticated user import logic
-        this.hasImportedUsers = true;
-    }
-    async prepareUpdateData(db, collection, importDef) {
-        // Implementation for preparing update data (preserve existing logic)
-        // This would handle the update logic from the original DataLoader
-    }
-    async executePostImportActions(dbId, specificCollections) {
-        // Implementation for executing post-import actions (preserve existing logic)
-        // This would handle file uploads and other post-import actions
-    }
-    async transferDataBetweenDatabases(sourceDb, targetDb) {
-        // Implementation for transferring data between databases (preserve existing logic)
-        // This would handle the existing transfer logic
-    }
-}

package/dist/migrations/services/RateLimitManager.d.ts

@@ -1,138 +0,0 @@
-export interface RateLimitConfig {
-    dataInsertion?: number;
-    dataUpdate?: number;
-    dataQuery?: number;
-    fileUpload?: number;
-    fileDownload?: number;
-    validation?: number;
-    relationshipResolution?: number;
-    userCreation?: number;
-    userUpdate?: number;
-    apiCalls?: number;
-}
-/**
- * Manages rate limiting across the import system.
- * Provides configurable p-limit instances for different operation types.
- * Builds on existing p-limit usage in attributeManager and other components.
- */
-export declare class RateLimitManager {
-    private limits;
-    private config;
-    private static readonly DEFAULT_LIMITS;
-    constructor(config?: Partial<RateLimitConfig>);
-    /**
-     * Initializes p-limit instances for all operation types.
-     */
-    private initializeLimits;
-    /**
-     * Gets the rate limiter for data insertion operations.
-     */
-    get dataInsertion(): any;
-    /**
-     * Gets the rate limiter for data update operations.
-     */
-    get dataUpdate(): any;
-    /**
-     * Gets the rate limiter for data query operations.
-     */
-    get dataQuery(): any;
-    /**
-     * Gets the rate limiter for file upload operations.
-     */
-    get fileUpload(): any;
-    /**
-     * Gets the rate limiter for file download operations.
-     */
-    get fileDownload(): any;
-    /**
-     * Gets the rate limiter for validation operations.
-     */
-    get validation(): any;
-    /**
-     * Gets the rate limiter for relationship resolution operations.
-     */
-    get relationshipResolution(): any;
-    /**
-     * Gets the rate limiter for user creation operations.
-     */
-    get userCreation(): any;
-    /**
-     * Gets the rate limiter for user update operations.
-     */
-    get userUpdate(): any;
-    /**
-     * Gets the rate limiter for general API calls.
-     */
-    get apiCalls(): any;
-    /**
-     * Gets a rate limiter by operation type name.
-     *
-     * @param operationType - The type of operation
-     * @returns The p-limit instance for the operation type
-     */
-    getLimiter(operationType: keyof RateLimitConfig): any;
-    /**
-     * Updates the rate limit for a specific operation type.
-     *
-     * @param operationType - The operation type to update
-     * @param newLimit - The new concurrent operation limit
-     */
-    updateLimit(operationType: keyof RateLimitConfig, newLimit: number): void;
-    /**
-     * Updates multiple rate limits at once.
-     *
-     * @param newConfig - Partial configuration with new limits
-     */
-    updateLimits(newConfig: Partial<RateLimitConfig>): void;
-    /**
-     * Gets the current configuration.
-     */
-    getConfig(): Required<RateLimitConfig>;
-    /**
-     * Gets statistics about pending and active operations.
-     * Useful for monitoring and debugging rate limiting.
-     */
-    getStatistics(): {
-        [operationType: string]: {
-            pending: number;
-            active: number;
-        };
-    };
-    /**
-     * Waits for all pending operations to complete.
-     * Useful for graceful shutdown or testing.
-     *
-     * @param timeout - Maximum time to wait in milliseconds (default: 30 seconds)
-     */
-    waitForCompletion(timeout?: number): Promise<void>;
-    /**
-     * Creates a rate limiter with automatic retry logic.
-     * Combines p-limit with retry functionality for robust operation handling.
-     *
-     * @param operationType - The operation type to get the limiter for
-     * @param maxRetries - Maximum number of retries (default: 3)
-     * @param retryDelay - Delay between retries in milliseconds (default: 1000)
-     */
-    createRetryLimiter(operationType: keyof RateLimitConfig, maxRetries?: number, retryDelay?: number): <T>(operation: () => Promise<T>) => Promise<T>;
-    /**
-     * Adjusts rate limits based on API response times and error rates.
-     * Implements adaptive rate limiting for optimal performance.
-     *
-     * @param operationType - The operation type to adjust
-     * @param responseTime - Average response time in milliseconds
-     * @param errorRate - Error rate as a percentage (0-100)
-     */
-    adaptiveAdjust(operationType: keyof RateLimitConfig, responseTime: number, errorRate: number): void;
-    /**
-     * Resets all rate limits to default values.
-     */
-    resetToDefaults(): void;
-    /**
-     * Creates a specialized batch processor with rate limiting.
-     * Useful for processing large datasets with controlled concurrency.
-     *
-     * @param operationType - The operation type for rate limiting
-     * @param batchSize - Number of items to process in each batch
-     */
-    createBatchProcessor<T, R>(operationType: keyof RateLimitConfig, batchSize?: number): (items: T[], processor: (item: T) => Promise<R>, onProgress?: (processed: number, total: number) => void) => Promise<R[]>;
-}