appwrite-utils-cli 0.10.85 → 1.0.1

package/dist/migrations/services/RateLimitManager.js

@@ -0,0 +1,279 @@
+import pLimit from "p-limit";
+import { logger } from "../../shared/logging.js";
+/**
+ * Manages rate limiting across the import system.
+ * Provides configurable p-limit instances for different operation types.
+ * Builds on existing p-limit usage in attributeManager and other components.
+ */
+export class RateLimitManager {
+    limits = new Map();
+    config;
+    // Default rate limits based on existing usage and best practices
+    static DEFAULT_LIMITS = {
+        dataInsertion: 5, // Conservative for database writes
+        dataUpdate: 8, // Slightly higher for updates
+        dataQuery: 25, // Higher for read operations (from existing queryLimit)
+        fileUpload: 2, // Very conservative for file uploads
+        fileDownload: 3, // Slightly higher for downloads
+        validation: 10, // Higher for validation operations
+        relationshipResolution: 8, // Moderate for relationship operations
+        userCreation: 3, // Conservative for user creation
+        userUpdate: 5, // Moderate for user updates
+        apiCalls: 15, // General API call limit
+    };
+    constructor(config) {
+        this.config = { ...RateLimitManager.DEFAULT_LIMITS, ...config };
+        this.initializeLimits();
+    }
+    /**
+     * Initializes p-limit instances for all operation types.
+     */
+    initializeLimits() {
+        for (const [operation, limit] of Object.entries(this.config)) {
+            this.limits.set(operation, pLimit(limit));
+            logger.debug(`Rate limit for ${operation}: ${limit} concurrent operations`);
+        }
+    }
+    /**
+     * Gets the rate limiter for data insertion operations.
+     */
+    get dataInsertion() {
+        return this.limits.get('dataInsertion');
+    }
+    /**
+     * Gets the rate limiter for data update operations.
+     */
+    get dataUpdate() {
+        return this.limits.get('dataUpdate');
+    }
+    /**
+     * Gets the rate limiter for data query operations.
+     */
+    get dataQuery() {
+        return this.limits.get('dataQuery');
+    }
+    /**
+     * Gets the rate limiter for file upload operations.
+     */
+    get fileUpload() {
+        return this.limits.get('fileUpload');
+    }
+    /**
+     * Gets the rate limiter for file download operations.
+     */
+    get fileDownload() {
+        return this.limits.get('fileDownload');
+    }
+    /**
+     * Gets the rate limiter for validation operations.
+     */
+    get validation() {
+        return this.limits.get('validation');
+    }
+    /**
+     * Gets the rate limiter for relationship resolution operations.
+     */
+    get relationshipResolution() {
+        return this.limits.get('relationshipResolution');
+    }
+    /**
+     * Gets the rate limiter for user creation operations.
+     */
+    get userCreation() {
+        return this.limits.get('userCreation');
+    }
+    /**
+     * Gets the rate limiter for user update operations.
+     */
+    get userUpdate() {
+        return this.limits.get('userUpdate');
+    }
+    /**
+     * Gets the rate limiter for general API calls.
+     */
+    get apiCalls() {
+        return this.limits.get('apiCalls');
+    }
+    /**
+     * Gets a rate limiter by operation type name.
+     *
+     * @param operationType - The type of operation
+     * @returns The p-limit instance for the operation type
+     */
+    getLimiter(operationType) {
+        const limiter = this.limits.get(operationType);
+        if (!limiter) {
+            logger.warn(`No rate limiter found for operation type: ${operationType}. Using default API calls limiter.`);
+            return this.limits.get('apiCalls');
+        }
+        return limiter;
+    }
+    /**
+     * Updates the rate limit for a specific operation type.
+     *
+     * @param operationType - The operation type to update
+     * @param newLimit - The new concurrent operation limit
+     */
+    updateLimit(operationType, newLimit) {
+        this.config[operationType] = newLimit;
+        this.limits.set(operationType, pLimit(newLimit));
+        logger.info(`Updated rate limit for ${operationType}: ${newLimit} concurrent operations`);
+    }
+    /**
+     * Updates multiple rate limits at once.
+     *
+     * @param newConfig - Partial configuration with new limits
+     */
+    updateLimits(newConfig) {
+        for (const [operation, limit] of Object.entries(newConfig)) {
+            if (limit !== undefined) {
+                this.updateLimit(operation, limit);
+            }
+        }
+    }
+    /**
+     * Gets the current configuration.
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Gets statistics about pending and active operations.
+     * Useful for monitoring and debugging rate limiting.
+     */
+    getStatistics() {
+        const stats = {};
+        for (const [operationType, limiter] of this.limits.entries()) {
+            stats[operationType] = {
+                pending: limiter.pendingCount,
+                active: limiter.activeCount,
+            };
+        }
+        return stats;
+    }
+    /**
+     * Waits for all pending operations to complete.
+     * Useful for graceful shutdown or testing.
+     *
+     * @param timeout - Maximum time to wait in milliseconds (default: 30 seconds)
+     */
+    async waitForCompletion(timeout = 30000) {
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            const stats = this.getStatistics();
+            const hasPendingOperations = Object.values(stats).some(stat => stat.pending > 0 || stat.active > 0);
+            if (!hasPendingOperations) {
+                logger.info("All rate-limited operations completed");
+                return;
+            }
+            // Log current status
+            const pendingOperations = Object.entries(stats)
+                .filter(([_, stat]) => stat.pending > 0 || stat.active > 0)
+                .map(([type, stat]) => `${type}: ${stat.pending} pending, ${stat.active} active`)
+                .join(", ");
+            logger.debug(`Waiting for operations to complete: ${pendingOperations}`);
+            // Wait a bit before checking again
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+        logger.warn(`Timeout waiting for rate-limited operations to complete after ${timeout}ms`);
+    }
+    /**
+     * Creates a rate limiter with automatic retry logic.
+     * Combines p-limit with retry functionality for robust operation handling.
+     *
+     * @param operationType - The operation type to get the limiter for
+     * @param maxRetries - Maximum number of retries (default: 3)
+     * @param retryDelay - Delay between retries in milliseconds (default: 1000)
+     */
+    createRetryLimiter(operationType, maxRetries = 3, retryDelay = 1000) {
+        const limiter = this.getLimiter(operationType);
+        return async (operation) => {
+            return limiter(async () => {
+                let lastError;
+                for (let attempt = 0; attempt <= maxRetries; attempt++) {
+                    try {
+                        return await operation();
+                    }
+                    catch (error) {
+                        lastError = error;
+                        if (attempt < maxRetries) {
+                            logger.warn(`Operation failed (attempt ${attempt + 1}/${maxRetries + 1}): ${lastError.message}. Retrying in ${retryDelay}ms...`);
+                            await new Promise(resolve => setTimeout(resolve, retryDelay));
+                        }
+                    }
+                }
+                // If we get here, all retries failed
+                logger.error(`Operation failed after ${maxRetries + 1} attempts: ${lastError?.message}`);
+                throw lastError;
+            });
+        };
+    }
+    /**
+     * Adjusts rate limits based on API response times and error rates.
+     * Implements adaptive rate limiting for optimal performance.
+     *
+     * @param operationType - The operation type to adjust
+     * @param responseTime - Average response time in milliseconds
+     * @param errorRate - Error rate as a percentage (0-100)
+     */
+    adaptiveAdjust(operationType, responseTime, errorRate) {
+        const currentLimit = this.config[operationType];
+        let newLimit = currentLimit;
+        // If error rate is high, reduce concurrency
+        if (errorRate > 10) {
+            newLimit = Math.max(1, Math.floor(currentLimit * 0.7));
+            logger.info(`Reducing ${operationType} limit due to high error rate (${errorRate}%): ${currentLimit} -> ${newLimit}`);
+        }
+        // If response time is high, reduce concurrency
+        else if (responseTime > 5000) {
+            newLimit = Math.max(1, Math.floor(currentLimit * 0.8));
+            logger.info(`Reducing ${operationType} limit due to high response time (${responseTime}ms): ${currentLimit} -> ${newLimit}`);
+        }
+        // If performance is good, gradually increase concurrency
+        else if (errorRate < 2 && responseTime < 1000 && currentLimit < RateLimitManager.DEFAULT_LIMITS[operationType] * 2) {
+            newLimit = Math.min(RateLimitManager.DEFAULT_LIMITS[operationType] * 2, currentLimit + 1);
+            logger.info(`Increasing ${operationType} limit due to good performance: ${currentLimit} -> ${newLimit}`);
+        }
+        if (newLimit !== currentLimit) {
+            this.updateLimit(operationType, newLimit);
+        }
+    }
+    /**
+     * Resets all rate limits to default values.
+     */
+    resetToDefaults() {
+        logger.info("Resetting all rate limits to default values");
+        this.config = { ...RateLimitManager.DEFAULT_LIMITS };
+        this.initializeLimits();
+    }
+    /**
+     * Creates a specialized batch processor with rate limiting.
+     * Useful for processing large datasets with controlled concurrency.
+     *
+     * @param operationType - The operation type for rate limiting
+     * @param batchSize - Number of items to process in each batch
+     */
+    createBatchProcessor(operationType, batchSize = 50) {
+        const limiter = this.getLimiter(operationType);
+        return async (items, processor, onProgress) => {
+            const results = [];
+            for (let i = 0; i < items.length; i += batchSize) {
+                const batch = items.slice(i, i + batchSize);
+                const batchPromises = batch.map(item => limiter(() => processor(item)));
+                const batchResults = await Promise.allSettled(batchPromises);
+                for (const result of batchResults) {
+                    if (result.status === 'fulfilled') {
+                        results.push(result.value);
+                    }
+                    else {
+                        logger.error(`Batch item failed: ${result.reason}`);
+                    }
+                }
+                if (onProgress) {
+                    onProgress(Math.min(i + batchSize, items.length), items.length);
+                }
+            }
+            return results;
+        };
+    }
+}

package/dist/migrations/services/RelationshipResolver.d.ts

@@ -0,0 +1,120 @@
+import type { AppwriteConfig, CollectionCreate } from "appwrite-utils";
+import type { UserMappingService } from "./UserMappingService.js";
+export interface CollectionImportData {
+    collection?: CollectionCreate;
+    data: Array<{
+        rawData: any;
+        finalData: any;
+        context: any;
+        importDef?: any;
+    }>;
+}
+/**
+ * Service responsible for resolving relationships and ID mappings during import.
+ * Preserves all existing relationship resolution logic from DataLoader.
+ * Extracted to provide focused, testable relationship management.
+ */
+export declare class RelationshipResolver {
+    private config;
+    private userMappingService;
+    private oldIdToNewIdPerCollectionMap;
+    constructor(config: AppwriteConfig, userMappingService: UserMappingService);
+    /**
+     * Helper method to generate a consistent key for collections.
+     * Preserves existing logic from DataLoader.
+     */
+    getCollectionKey(name: string): string;
+    /**
+     * Stores ID mapping for a collection.
+     *
+     * @param collectionName - The collection name
+     * @param oldId - The old ID
+     * @param newId - The new ID
+     */
+    setIdMapping(collectionName: string, oldId: string, newId: string): void;
+    /**
+     * Gets the new ID for an old ID in a specific collection.
+     *
+     * @param collectionName - The collection name
+     * @param oldId - The old ID to look up
+     * @returns The new ID if found, otherwise undefined
+     */
+    getNewIdForOldId(collectionName: string, oldId: string): string | undefined;
+    /**
+     * Checks if an ID mapping exists for a collection.
+     *
+     * @param collectionName - The collection name
+     * @param oldId - The old ID to check
+     * @returns True if mapping exists
+     */
+    hasIdMapping(collectionName: string, oldId: string): boolean;
+    /**
+     * Gets the value to match for a given key in the final data or context.
+     * Preserves existing logic from DataLoader.
+     *
+     * @param finalData - The final data object
+     * @param context - The context object
+     * @param key - The key to get the value for
+     * @returns The value to match for from finalData or Context
+     */
+    private getValueFromData;
+    /**
+     * Updates old references with new IDs based on ID mappings.
+     * Preserves existing reference update logic from DataLoader.
+     *
+     * @param importMap - Map of collection data by collection key
+     * @param collections - Array of collection configurations
+     */
+    updateOldReferencesForNew(importMap: Map<string, CollectionImportData>, collections: CollectionCreate[]): void;
+    /**
+     * Resolves relationship references using the merged user map.
+     * Handles cases where users have been merged during deduplication.
+     *
+     * @param oldId - The old user ID to resolve
+     * @param targetCollection - The target collection name
+     * @returns The resolved ID (new user ID if merged, otherwise mapped ID)
+     */
+    private getMergedId;
+    /**
+     * Validates relationship integrity before import.
+     * Checks that all referenced collections and fields exist.
+     *
+     * @param collections - Array of collections to validate
+     * @returns Array of validation errors
+     */
+    validateRelationships(collections: CollectionCreate[]): string[];
+    /**
+     * Gets relationship statistics for reporting.
+     *
+     * @returns Statistics about relationships and ID mappings
+     */
+    getStatistics(): {
+        totalCollections: number;
+        totalIdMappings: number;
+        collectionsWithMappings: string[];
+    };
+    /**
+     * Clears all ID mappings (useful for testing or resetting state).
+     */
+    clearAllMappings(): void;
+    /**
+     * Exports ID mappings for debugging or external use.
+     *
+     * @returns Serializable object containing all ID mappings
+     */
+    exportMappings(): {
+        [collectionKey: string]: {
+            [oldId: string]: string;
+        };
+    };
+    /**
+     * Imports ID mappings from an external source.
+     *
+     * @param mappings - The mappings to import
+     */
+    importMappings(mappings: {
+        [collectionKey: string]: {
+            [oldId: string]: string;
+        };
+    }): void;
+}