appwrite-utils-cli 1.11.0 → 1.12.0

package/dist/migrations/services/FileHandlerService.js

@@ -0,0 +1,236 @@
+import path from "path";
+import fs from "fs";
+import { logger } from 'appwrite-utils-helpers';
+import { RateLimitManager } from "./RateLimitManager.js";
+/**
+ * Service responsible for file handling during import.
+ * Preserves all existing file handling capabilities including URL support.
+ * Extracted from DataLoader to provide focused, testable file operations.
+ */
+export class FileHandlerService {
+    appwriteFolderPath;
+    config;
+    importDataActions;
+    rateLimitManager;
+    constructor(appwriteFolderPath, config, importDataActions, rateLimitManager) {
+        this.appwriteFolderPath = appwriteFolderPath;
+        this.config = config;
+        this.importDataActions = importDataActions;
+        this.rateLimitManager = rateLimitManager || new RateLimitManager();
+    }
+    /**
+     * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+     * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+     * and update the field with the file's ID if necessary.
+     *
+     * Preserves existing file handling logic from DataLoader.
+     *
+     * @param attributeMappings - The attribute mappings from the import definition.
+     * @param context - The context object containing information about the database, collection, and document.
+     * @param item - The item being imported, used for resolving template paths in fileData mappings.
+     * @returns The attribute mappings updated with any necessary post-import actions.
+     */
+    getAttributeMappingsWithFileActions(attributeMappings, context, item) {
+        // Iterate over each attribute mapping to check for fileData attributes
+        return attributeMappings.map((mapping) => {
+            if (mapping.fileData) {
+                // Resolve the file path using the provided template, context, and item
+                let mappingFilePath = this.importDataActions.resolveTemplate(mapping.fileData.path, context, item);
+                // Ensure the file path is absolute if it doesn't start with "http"
+                if (!mappingFilePath.toLowerCase().startsWith("http")) {
+                    mappingFilePath = this.resolveLocalFilePath(mappingFilePath);
+                }
+                // Define the after-import action to create a file and update the field
+                const afterImportAction = {
+                    action: "createFileAndUpdateField",
+                    params: [
+                        "{dbId}",
+                        "{collId}",
+                        "{docId}",
+                        mapping.targetKey,
+                        `${this.config.documentBucketId}_${context.dbName
+                            .toLowerCase()
+                            .replace(" ", "")}`, // Bucket ID pattern
+                        mappingFilePath,
+                        mapping.fileData.name,
+                    ],
+                };
+                // Add the after-import action to the mapping's postImportActions array
+                const postImportActions = mapping.postImportActions
+                    ? [...mapping.postImportActions, afterImportAction]
+                    : [afterImportAction];
+                return { ...mapping, postImportActions };
+            }
+            // Return the mapping unchanged if no fileData attribute is found
+            return mapping;
+        });
+    }
+    /**
+     * Resolves local file path, searching in subdirectories if needed.
+     * Preserves existing file search logic from DataLoader.
+     *
+     * @param mappingFilePath - The relative file path from the mapping
+     * @returns Resolved absolute file path
+     */
+    resolveLocalFilePath(mappingFilePath) {
+        // First try the direct path
+        let fullPath = path.resolve(this.appwriteFolderPath, mappingFilePath);
+        // If file doesn't exist, search in subdirectories
+        if (!fs.existsSync(fullPath)) {
+            const findFileInDir = (dir) => {
+                try {
+                    const files = fs.readdirSync(dir);
+                    for (const file of files) {
+                        const filePath = path.join(dir, file);
+                        const stat = fs.statSync(filePath);
+                        if (stat.isDirectory()) {
+                            // Recursively search subdirectories
+                            const found = findFileInDir(filePath);
+                            if (found)
+                                return found;
+                        }
+                        else if (file === path.basename(mappingFilePath)) {
+                            return filePath;
+                        }
+                    }
+                }
+                catch (error) {
+                    logger.warn(`Error reading directory ${dir}: ${error}`);
+                }
+                return null;
+            };
+            const foundPath = findFileInDir(this.appwriteFolderPath);
+            if (foundPath) {
+                mappingFilePath = foundPath;
+            }
+            else {
+                logger.warn(`File not found in any subdirectory: ${mappingFilePath}`);
+                // Keep the original resolved path as fallback
+                mappingFilePath = fullPath;
+            }
+        }
+        else {
+            mappingFilePath = fullPath;
+        }
+        return mappingFilePath;
+    }
+    /**
+     * Executes file-related post-import actions with rate limiting.
+     * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
+     *
+     * @param context - The context containing document and collection information
+     * @param postImportActions - The post-import actions to execute
+     */
+    async executeFileActions(context, postImportActions) {
+        const fileActions = postImportActions.filter(action => action.action === "createFileAndUpdateField");
+        if (fileActions.length === 0) {
+            return;
+        }
+        // Execute file actions with rate limiting
+        const filePromises = fileActions.map(action => this.rateLimitManager.fileUpload(() => this.executeFileAction(action, context)));
+        try {
+            await Promise.allSettled(filePromises);
+        }
+        catch (error) {
+            logger.error(`Error executing file actions for context: ${JSON.stringify(context, null, 2)}`, error);
+        }
+    }
+    /**
+     * Executes a single file action with proper error handling.
+     *
+     * @param action - The file action to execute
+     * @param context - The context for template resolution
+     */
+    async executeFileAction(action, context) {
+        try {
+            // Resolve parameters using existing template resolution
+            const resolvedParams = action.params.map((param) => this.importDataActions.resolveTemplate(param, context, context));
+            // Execute the createFileAndUpdateField action
+            await this.importDataActions.executeAction(action.action, resolvedParams, context, context);
+        }
+        catch (error) {
+            logger.error(`Failed to execute file action '${action.action}' with params: ${JSON.stringify(action.params, null, 2)}`, error);
+            // Don't throw - we want to continue with other file operations
+        }
+    }
+    /**
+     * Validates that file paths exist before import begins.
+     * Provides early validation to catch file issues before processing starts.
+     *
+     * @param attributeMappings - The attribute mappings to validate
+     * @param context - The context for template resolution
+     * @param item - The item for template resolution
+     * @returns Array of validation errors (empty if all valid)
+     */
+    validateFilePaths(attributeMappings, context, item) {
+        const errors = [];
+        for (const mapping of attributeMappings) {
+            if (mapping.fileData) {
+                try {
+                    let filePath = this.importDataActions.resolveTemplate(mapping.fileData.path, context, item);
+                    // Skip URL validation (URLs are handled by the action itself)
+                    if (filePath.toLowerCase().startsWith("http")) {
+                        continue;
+                    }
+                    // Check if local file exists
+                    const resolvedPath = this.resolveLocalFilePath(filePath);
+                    if (!fs.existsSync(resolvedPath)) {
+                        errors.push(`File not found for mapping '${mapping.targetKey}': ${resolvedPath}`);
+                    }
+                }
+                catch (error) {
+                    errors.push(`Error validating file path for mapping '${mapping.targetKey}': ${error}`);
+                }
+            }
+        }
+        return errors;
+    }
+    /**
+     * Gets file statistics for import planning.
+     * Helps estimate import time and resource requirements.
+     *
+     * @param attributeMappings - The attribute mappings to analyze
+     * @param items - The items that will be imported
+     * @returns File statistics object
+     */
+    getFileStatistics(attributeMappings, items) {
+        let totalFiles = 0;
+        let urlFiles = 0;
+        let localFiles = 0;
+        let estimatedSize = 0;
+        const fileAttributeMappings = attributeMappings.filter(mapping => mapping.fileData);
+        for (const item of items) {
+            for (const mapping of fileAttributeMappings) {
+                totalFiles++;
+                try {
+                    const filePath = this.importDataActions.resolveTemplate(mapping.fileData.path, {}, item);
+                    if (filePath.toLowerCase().startsWith("http")) {
+                        urlFiles++;
+                    }
+                    else {
+                        localFiles++;
+                        try {
+                            const resolvedPath = this.resolveLocalFilePath(filePath);
+                            if (fs.existsSync(resolvedPath)) {
+                                const stats = fs.statSync(resolvedPath);
+                                estimatedSize += stats.size;
+                            }
+                        }
+                        catch (error) {
+                            // Ignore errors for statistics
+                        }
+                    }
+                }
+                catch (error) {
+                    // Ignore errors for statistics
+                }
+            }
+        }
+        return {
+            totalFiles,
+            urlFiles,
+            localFiles,
+            estimatedSize
+        };
+    }
+}

package/dist/migrations/services/ImportOrchestrator.d.ts

@@ -0,0 +1,99 @@
+import { type Databases, type Models, type Storage } from "node-appwrite";
+import type { AppwriteConfig } from "appwrite-utils";
+import { type RateLimitConfig } from "./RateLimitManager.js";
+import type { ImportDataActions } from "../importDataActions.js";
+import type { SetupOptions } from "../../utilsController.js";
+/**
+ * Orchestrator for the entire import process.
+ * Coordinates all services while preserving existing functionality and performance characteristics.
+ *
+ * This replaces the monolithic ImportController and DataLoader with a cleaner, modular architecture.
+ */
+export declare class ImportOrchestrator {
+    private config;
+    private database;
+    private storage;
+    private appwriteFolderPath;
+    private setupOptions;
+    private databasesToRun;
+    private dataTransformationService;
+    private fileHandlerService;
+    private userMappingService;
+    private validationService;
+    private relationshipResolver;
+    private rateLimitManager;
+    private importMap;
+    private collectionImportOperations;
+    private hasImportedUsers;
+    private batchLimit;
+    private _adapter;
+    private getAdapter;
+    constructor(config: AppwriteConfig, database: Databases, storage: Storage, appwriteFolderPath: string, importDataActions: ImportDataActions, setupOptions: SetupOptions, databasesToRun?: Models.Database[], rateLimitConfig?: Partial<RateLimitConfig>);
+    /**
+     * Main entry point for the import process.
+     * Preserves existing import flow while using the new modular architecture.
+     */
+    run(specificCollections?: string[]): Promise<void>;
+    /**
+     * Performs the complete import process for a single database.
+     */
+    private performDatabaseImport;
+    /**
+     * Sets up import maps and operation tracking.
+     * Preserves existing setup logic from DataLoader.
+     */
+    private setupImportMaps;
+    /**
+     * Loads existing users and initializes user mapping service.
+     */
+    private loadExistingUsers;
+    /**
+     * Loads and prepares data for all collections.
+     */
+    private loadAndPrepareData;
+    /**
+     * Prepares data for a regular collection.
+     * Uses the DataTransformationService for all transformations.
+     */
+    private prepareCollectionData;
+    /**
+     * Prepares data for user collection with deduplication.
+     * Uses the UserMappingService for sophisticated user handling.
+     */
+    private prepareUserCollectionData;
+    /**
+     * Imports collections with rate limiting and batch processing.
+     * Preserves existing import logic with enhanced error handling.
+     */
+    private importCollections;
+    /**
+     * Imports a single collection with batching and rate limiting.
+     */
+    private importSingleCollection;
+    /**
+     * Imports a single item with error handling.
+     */
+    private importSingleItem;
+    /**
+     * Helper method to generate consistent collection keys.
+     */
+    private getCollectionKey;
+    /**
+     * Loads data from file based on import definition.
+     */
+    private loadDataFromFile;
+    /**
+     * Creates batches for processing with the specified batch size.
+     */
+    private createBatches;
+    /**
+     * Generates a unique ID for documents.
+     */
+    private generateUniqueId;
+    private findExistingCollection;
+    private updateOperationStatus;
+    private importUsersCollection;
+    private prepareUpdateData;
+    private executePostImportActions;
+    private transferDataBetweenDatabases;
+}