appwrite-utils-cli 0.10.86 → 1.0.2

package/src/migrations/services/DataTransformationService.ts

@@ -0,0 +1,196 @@
+import type {
+  AttributeMappings,
+  ImportDef,
+  ConfigDatabase,
+  CollectionCreate,
+} from "appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+import { convertObjectByAttributeMappings } from "../../utils/dataConverters.js";
+import { logger } from "../../shared/logging.js";
+
+/**
+ * Service responsible for data transformation during import.
+ * Extracted from DataLoader to provide focused, testable transformation logic.
+ */
+export class DataTransformationService {
+  private importDataActions: ImportDataActions;
+
+  constructor(importDataActions: ImportDataActions) {
+    this.importDataActions = importDataActions;
+  }
+
+  /**
+   * Transforms the given item based on the provided attribute mappings.
+   * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+   * 
+   * Preserves existing transformation logic from DataLoader.
+   *
+   * @param item - The item to be transformed.
+   * @param attributeMappings - The mappings that define how each attribute should be transformed.
+   * @returns The transformed item.
+   */
+  transformData(item: any, attributeMappings: AttributeMappings): any {
+    try {
+      // Convert the item using the attribute mappings provided
+      const convertedItem = convertObjectByAttributeMappings(
+        item,
+        attributeMappings
+      );
+      
+      // Run additional converter functions on the converted item, if any
+      return this.importDataActions.runConverterFunctions(
+        convertedItem,
+        attributeMappings
+      );
+    } catch (error) {
+      logger.error(`Error transforming data for item: ${JSON.stringify(item, null, 2)}`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Creates a context object for data transformation.
+   * Preserves existing context creation logic from DataLoader.
+   *
+   * @param db - The database configuration
+   * @param collection - The collection configuration
+   * @param item - The raw item data
+   * @param docId - The document ID
+   * @returns Context object for transformation
+   */
+  createContext(
+    db: ConfigDatabase,
+    collection: CollectionCreate,
+    item: any,
+    docId: string
+  ) {
+    return {
+      ...item, // Spread the item data for easy access to its properties
+      dbId: db.$id,
+      dbName: db.name,
+      collId: collection.$id,
+      collName: collection.name,
+      docId: docId,
+      createdDoc: {}, // Initially empty, to be updated when the document is created
+    };
+  }
+
+  /**
+   * Merges two objects by updating the source object with the target object's values.
+   * Preserves existing merge logic from DataLoader.
+   * 
+   * It iterates through the target object's keys and updates the source object if:
+   * - The source object has the key.
+   * - The target object's value for that key is not null, undefined, or an empty string.
+   * - If the target object has an array value, it concatenates the values and removes duplicates.
+   *
+   * @param source - The source object to be updated.
+   * @param update - The target object with values to update the source object.
+   * @returns The updated source object.
+   */
+  mergeObjects(source: any, update: any): any {
+    // Create a new object to hold the merged result
+    const result = { ...source };
+
+    // Loop through the keys of the object we care about
+    for (const [key, value] of Object.entries(source)) {
+      // Check if the key exists in the target object
+      if (!Object.hasOwn(update, key)) {
+        // If the key doesn't exist, we can just skip it
+        continue;
+      }
+      if (update[key] === value) {
+        continue;
+      }
+      // If the value ain't here, we can just do whatever man
+      if (value === undefined || value === null || value === "") {
+        // If the update key is defined
+        if (
+          update[key] !== undefined &&
+          update[key] !== null &&
+          update[key] !== ""
+        ) {
+          // might as well use it eh?
+          result[key] = update[key];
+        }
+        // ELSE if the value is an array, because it would then not be === to those things above
+      } else if (Array.isArray(value)) {
+        // Get the update value
+        const updateValue = update[key];
+        // If the update value is an array, concatenate and remove duplicates
+        // and poopy data
+        if (Array.isArray(updateValue)) {
+          result[key] = [...new Set([...value, ...updateValue])].filter(
+            (item) => item !== null && item !== undefined && item !== ""
+          );
+        } else {
+          // If the update value is not an array, just use it
+          result[key] = [...value, updateValue].filter(
+            (item) => item !== null && item !== undefined && item !== ""
+          );
+        }
+      } else if (typeof value === "object" && !Array.isArray(value)) {
+        // If the value is an object, we need to merge it
+        if (typeof update[key] === "object" && !Array.isArray(update[key])) {
+          result[key] = this.mergeObjects(value, update[key]);
+        }
+      } else {
+        // Finally, the source value is defined, and not an array, so we don't care about the update value
+        continue;
+      }
+    }
+    
+    // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+    for (const [key, value] of Object.entries(update)) {
+      if (value === undefined || value === null || value === "") {
+        continue;
+      } else if (!Object.hasOwn(source, key)) {
+        result[key] = value;
+      } else if (
+        typeof source[key] === "object" &&
+        typeof value === "object" &&
+        !Array.isArray(source[key]) &&
+        !Array.isArray(value)
+      ) {
+        result[key] = this.mergeObjects(source[key], value);
+      } else if (Array.isArray(source[key]) && Array.isArray(value)) {
+        result[key] = [...new Set([...source[key], ...value])].filter(
+          (item) => item !== null && item !== undefined && item !== ""
+        );
+      } else if (
+        source[key] === undefined ||
+        source[key] === null ||
+        source[key] === ""
+      ) {
+        result[key] = value;
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Validates the transformed data item using existing validation logic.
+   * 
+   * @param transformedData - The transformed data to validate
+   * @param attributeMappings - The attribute mappings containing validation rules
+   * @param context - The context for validation
+   * @returns True if valid, false otherwise
+   */
+  validateTransformedData(
+    transformedData: any,
+    attributeMappings: AttributeMappings,
+    context: any
+  ): boolean {
+    try {
+      return this.importDataActions.validateItem(
+        transformedData,
+        attributeMappings,
+        context
+      );
+    } catch (error) {
+      logger.error(`Validation error for transformed data: ${JSON.stringify(transformedData, null, 2)}`, error);
+      return false;
+    }
+  }
+}

package/src/migrations/services/FileHandlerService.ts

@@ -0,0 +1,311 @@
+import path from "path";
+import fs from "fs";
+import type { AttributeMappings, AppwriteConfig } from "appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+import { logger } from "../../shared/logging.js";
+import { RateLimitManager } from "./RateLimitManager.js";
+
+/**
+ * Service responsible for file handling during import.
+ * Preserves all existing file handling capabilities including URL support.
+ * Extracted from DataLoader to provide focused, testable file operations.
+ */
+export class FileHandlerService {
+  private appwriteFolderPath: string;
+  private config: AppwriteConfig;
+  private importDataActions: ImportDataActions;
+  private rateLimitManager: RateLimitManager;
+
+  constructor(
+    appwriteFolderPath: string,
+    config: AppwriteConfig,
+    importDataActions: ImportDataActions,
+    rateLimitManager?: RateLimitManager
+  ) {
+    this.appwriteFolderPath = appwriteFolderPath;
+    this.config = config;
+    this.importDataActions = importDataActions;
+    this.rateLimitManager = rateLimitManager || new RateLimitManager();
+  }
+
+  /**
+   * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+   * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+   * and update the field with the file's ID if necessary.
+   * 
+   * Preserves existing file handling logic from DataLoader.
+   *
+   * @param attributeMappings - The attribute mappings from the import definition.
+   * @param context - The context object containing information about the database, collection, and document.
+   * @param item - The item being imported, used for resolving template paths in fileData mappings.
+   * @returns The attribute mappings updated with any necessary post-import actions.
+   */
+  getAttributeMappingsWithFileActions(
+    attributeMappings: AttributeMappings,
+    context: any,
+    item: any
+  ): AttributeMappings {
+    // Iterate over each attribute mapping to check for fileData attributes
+    return attributeMappings.map((mapping) => {
+      if (mapping.fileData) {
+        // Resolve the file path using the provided template, context, and item
+        let mappingFilePath = this.importDataActions.resolveTemplate(
+          mapping.fileData.path,
+          context,
+          item
+        );
+        
+        // Ensure the file path is absolute if it doesn't start with "http"
+        if (!mappingFilePath.toLowerCase().startsWith("http")) {
+          mappingFilePath = this.resolveLocalFilePath(mappingFilePath);
+        }
+        
+        // Define the after-import action to create a file and update the field
+        const afterImportAction = {
+          action: "createFileAndUpdateField",
+          params: [
+            "{dbId}",
+            "{collId}",
+            "{docId}",
+            mapping.targetKey,
+            `${this.config!.documentBucketId}_${context.dbName
+              .toLowerCase()
+              .replace(" ", "")}`, // Bucket ID pattern
+            mappingFilePath,
+            mapping.fileData.name,
+          ],
+        };
+        
+        // Add the after-import action to the mapping's postImportActions array
+        const postImportActions = mapping.postImportActions
+          ? [...mapping.postImportActions, afterImportAction]
+          : [afterImportAction];
+        
+        return { ...mapping, postImportActions };
+      }
+      
+      // Return the mapping unchanged if no fileData attribute is found
+      return mapping;
+    });
+  }
+
+  /**
+   * Resolves local file path, searching in subdirectories if needed.
+   * Preserves existing file search logic from DataLoader.
+   * 
+   * @param mappingFilePath - The relative file path from the mapping
+   * @returns Resolved absolute file path
+   */
+  private resolveLocalFilePath(mappingFilePath: string): string {
+    // First try the direct path
+    let fullPath = path.resolve(this.appwriteFolderPath, mappingFilePath);
+
+    // If file doesn't exist, search in subdirectories
+    if (!fs.existsSync(fullPath)) {
+      const findFileInDir = (dir: string): string | null => {
+        try {
+          const files = fs.readdirSync(dir);
+
+          for (const file of files) {
+            const filePath = path.join(dir, file);
+            const stat = fs.statSync(filePath);
+
+            if (stat.isDirectory()) {
+              // Recursively search subdirectories
+              const found = findFileInDir(filePath);
+              if (found) return found;
+            } else if (file === path.basename(mappingFilePath)) {
+              return filePath;
+            }
+          }
+        } catch (error) {
+          logger.warn(`Error reading directory ${dir}: ${error}`);
+        }
+        return null;
+      };
+
+      const foundPath = findFileInDir(this.appwriteFolderPath);
+      if (foundPath) {
+        mappingFilePath = foundPath;
+      } else {
+        logger.warn(
+          `File not found in any subdirectory: ${mappingFilePath}`
+        );
+        // Keep the original resolved path as fallback
+        mappingFilePath = fullPath;
+      }
+    } else {
+      mappingFilePath = fullPath;
+    }
+
+    return mappingFilePath;
+  }
+
+  /**
+   * Executes file-related post-import actions with rate limiting.
+   * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
+   * 
+   * @param context - The context containing document and collection information
+   * @param postImportActions - The post-import actions to execute
+   */
+  async executeFileActions(
+    context: any,
+    postImportActions: any[]
+  ): Promise<void> {
+    const fileActions = postImportActions.filter(
+      action => action.action === "createFileAndUpdateField"
+    );
+
+    if (fileActions.length === 0) {
+      return;
+    }
+
+    // Execute file actions with rate limiting
+    const filePromises = fileActions.map(action =>
+      this.rateLimitManager.fileUpload(() => this.executeFileAction(action, context))
+    );
+
+    try {
+      await Promise.allSettled(filePromises);
+    } catch (error) {
+      logger.error(`Error executing file actions for context: ${JSON.stringify(context, null, 2)}`, error);
+    }
+  }
+
+  /**
+   * Executes a single file action with proper error handling.
+   * 
+   * @param action - The file action to execute
+   * @param context - The context for template resolution
+   */
+  private async executeFileAction(action: any, context: any): Promise<void> {
+    try {
+      // Resolve parameters using existing template resolution
+      const resolvedParams = action.params.map((param: any) =>
+        this.importDataActions.resolveTemplate(param, context, context)
+      );
+
+      // Execute the createFileAndUpdateField action
+      await this.importDataActions.executeAction(
+        action.action,
+        resolvedParams,
+        context,
+        context
+      );
+    } catch (error) {
+      logger.error(
+        `Failed to execute file action '${action.action}' with params: ${JSON.stringify(action.params, null, 2)}`,
+        error
+      );
+      // Don't throw - we want to continue with other file operations
+    }
+  }
+
+  /**
+   * Validates that file paths exist before import begins.
+   * Provides early validation to catch file issues before processing starts.
+   * 
+   * @param attributeMappings - The attribute mappings to validate
+   * @param context - The context for template resolution
+   * @param item - The item for template resolution
+   * @returns Array of validation errors (empty if all valid)
+   */
+  validateFilePaths(
+    attributeMappings: AttributeMappings,
+    context: any,
+    item: any
+  ): string[] {
+    const errors: string[] = [];
+
+    for (const mapping of attributeMappings) {
+      if (mapping.fileData) {
+        try {
+          let filePath = this.importDataActions.resolveTemplate(
+            mapping.fileData.path,
+            context,
+            item
+          );
+
+          // Skip URL validation (URLs are handled by the action itself)
+          if (filePath.toLowerCase().startsWith("http")) {
+            continue;
+          }
+
+          // Check if local file exists
+          const resolvedPath = this.resolveLocalFilePath(filePath);
+          if (!fs.existsSync(resolvedPath)) {
+            errors.push(
+              `File not found for mapping '${mapping.targetKey}': ${resolvedPath}`
+            );
+          }
+        } catch (error) {
+          errors.push(
+            `Error validating file path for mapping '${mapping.targetKey}': ${error}`
+          );
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Gets file statistics for import planning.
+   * Helps estimate import time and resource requirements.
+   * 
+   * @param attributeMappings - The attribute mappings to analyze
+   * @param items - The items that will be imported
+   * @returns File statistics object
+   */
+  getFileStatistics(attributeMappings: AttributeMappings, items: any[]): {
+    totalFiles: number;
+    urlFiles: number;
+    localFiles: number;
+    estimatedSize: number;
+  } {
+    let totalFiles = 0;
+    let urlFiles = 0;
+    let localFiles = 0;
+    let estimatedSize = 0;
+
+    const fileAttributeMappings = attributeMappings.filter(mapping => mapping.fileData);
+    
+    for (const item of items) {
+      for (const mapping of fileAttributeMappings) {
+        totalFiles++;
+        
+        try {
+          const filePath = this.importDataActions.resolveTemplate(
+            mapping.fileData!.path,
+            {},
+            item
+          );
+
+          if (filePath.toLowerCase().startsWith("http")) {
+            urlFiles++;
+          } else {
+            localFiles++;
+            try {
+              const resolvedPath = this.resolveLocalFilePath(filePath);
+              if (fs.existsSync(resolvedPath)) {
+                const stats = fs.statSync(resolvedPath);
+                estimatedSize += stats.size;
+              }
+            } catch (error) {
+              // Ignore errors for statistics
+            }
+          }
+        } catch (error) {
+          // Ignore errors for statistics
+        }
+      }
+    }
+
+    return {
+      totalFiles,
+      urlFiles,
+      localFiles,
+      estimatedSize
+    };
+  }
+}