appwrite-utils-cli 1.9.7 → 1.12.0

package/src/migrations/services/DataTransformationService.ts

@@ -1,196 +0,0 @@
-import type {
-  AttributeMappings,
-  ImportDef,
-  ConfigDatabase,
-  CollectionCreate,
-} from "appwrite-utils";
-import type { ImportDataActions } from "../importDataActions.js";
-import { convertObjectByAttributeMappings } from "../../utils/dataConverters.js";
-import { logger } from "../../shared/logging.js";
-
-/**
- * Service responsible for data transformation during import.
- * Extracted from DataLoader to provide focused, testable transformation logic.
- */
-export class DataTransformationService {
-  private importDataActions: ImportDataActions;
-
-  constructor(importDataActions: ImportDataActions) {
-    this.importDataActions = importDataActions;
-  }
-
-  /**
-   * Transforms the given item based on the provided attribute mappings.
-   * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
-   * 
-   * Preserves existing transformation logic from DataLoader.
-   *
-   * @param item - The item to be transformed.
-   * @param attributeMappings - The mappings that define how each attribute should be transformed.
-   * @returns The transformed item.
-   */
-  transformData(item: any, attributeMappings: AttributeMappings): any {
-    try {
-      // Convert the item using the attribute mappings provided
-      const convertedItem = convertObjectByAttributeMappings(
-        item,
-        attributeMappings
-      );
-      
-      // Run additional converter functions on the converted item, if any
-      return this.importDataActions.runConverterFunctions(
-        convertedItem,
-        attributeMappings
-      );
-    } catch (error) {
-      logger.error(`Error transforming data for item: ${JSON.stringify(item, null, 2)}`, error);
-      throw error;
-    }
-  }
-
-  /**
-   * Creates a context object for data transformation.
-   * Preserves existing context creation logic from DataLoader.
-   *
-   * @param db - The database configuration
-   * @param collection - The collection configuration
-   * @param item - The raw item data
-   * @param docId - The document ID
-   * @returns Context object for transformation
-   */
-  createContext(
-    db: ConfigDatabase,
-    collection: CollectionCreate,
-    item: any,
-    docId: string
-  ) {
-    return {
-      ...item, // Spread the item data for easy access to its properties
-      dbId: db.$id,
-      dbName: db.name,
-      collId: collection.$id,
-      collName: collection.name,
-      docId: docId,
-      createdDoc: {}, // Initially empty, to be updated when the document is created
-    };
-  }
-
-  /**
-   * Merges two objects by updating the source object with the target object's values.
-   * Preserves existing merge logic from DataLoader.
-   * 
-   * It iterates through the target object's keys and updates the source object if:
-   * - The source object has the key.
-   * - The target object's value for that key is not null, undefined, or an empty string.
-   * - If the target object has an array value, it concatenates the values and removes duplicates.
-   *
-   * @param source - The source object to be updated.
-   * @param update - The target object with values to update the source object.
-   * @returns The updated source object.
-   */
-  mergeObjects(source: any, update: any): any {
-    // Create a new object to hold the merged result
-    const result = { ...source };
-
-    // Loop through the keys of the object we care about
-    for (const [key, value] of Object.entries(source)) {
-      // Check if the key exists in the target object
-      if (!Object.hasOwn(update, key)) {
-        // If the key doesn't exist, we can just skip it
-        continue;
-      }
-      if (update[key] === value) {
-        continue;
-      }
-      // If the value ain't here, we can just do whatever man
-      if (value === undefined || value === null || value === "") {
-        // If the update key is defined
-        if (
-          update[key] !== undefined &&
-          update[key] !== null &&
-          update[key] !== ""
-        ) {
-          // might as well use it eh?
-          result[key] = update[key];
-        }
-        // ELSE if the value is an array, because it would then not be === to those things above
-      } else if (Array.isArray(value)) {
-        // Get the update value
-        const updateValue = update[key];
-        // If the update value is an array, concatenate and remove duplicates
-        // and poopy data
-        if (Array.isArray(updateValue)) {
-          result[key] = [...new Set([...value, ...updateValue])].filter(
-            (item) => item !== null && item !== undefined && item !== ""
-          );
-        } else {
-          // If the update value is not an array, just use it
-          result[key] = [...value, updateValue].filter(
-            (item) => item !== null && item !== undefined && item !== ""
-          );
-        }
-      } else if (typeof value === "object" && !Array.isArray(value)) {
-        // If the value is an object, we need to merge it
-        if (typeof update[key] === "object" && !Array.isArray(update[key])) {
-          result[key] = this.mergeObjects(value, update[key]);
-        }
-      } else {
-        // Finally, the source value is defined, and not an array, so we don't care about the update value
-        continue;
-      }
-    }
-    
-    // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
-    for (const [key, value] of Object.entries(update)) {
-      if (value === undefined || value === null || value === "") {
-        continue;
-      } else if (!Object.hasOwn(source, key)) {
-        result[key] = value;
-      } else if (
-        typeof source[key] === "object" &&
-        typeof value === "object" &&
-        !Array.isArray(source[key]) &&
-        !Array.isArray(value)
-      ) {
-        result[key] = this.mergeObjects(source[key], value);
-      } else if (Array.isArray(source[key]) && Array.isArray(value)) {
-        result[key] = [...new Set([...source[key], ...value])].filter(
-          (item) => item !== null && item !== undefined && item !== ""
-        );
-      } else if (
-        source[key] === undefined ||
-        source[key] === null ||
-        source[key] === ""
-      ) {
-        result[key] = value;
-      }
-    }
-
-    return result;
-  }
-
-  /**
-   * Validates the transformed data item using existing validation logic.
-   * 
-   * @param transformedData - The transformed data to validate
-   * @param attributeMappings - The attribute mappings containing validation rules
-   * @param context - The context for validation
-   * @returns True if valid, false otherwise
-   */
-  validateTransformedData(
-    transformedData: any,
-    attributeMappings: AttributeMappings,
-    context: any
-  ): boolean {
-    try {
-      return this.importDataActions.validateItem(
-        transformedData,
-        attributeMappings,
-        context
-      );
-    } catch (error) {
-      logger.error(`Validation error for transformed data: ${JSON.stringify(transformedData, null, 2)}`, error);
-      return false;
-    }
-  }
-}

package/src/migrations/services/FileHandlerService.ts

@@ -1,311 +0,0 @@
-import path from "path";
-import fs from "fs";
-import type { AttributeMappings, AppwriteConfig } from "appwrite-utils";
-import type { ImportDataActions } from "../importDataActions.js";
-import { logger } from "../../shared/logging.js";
-import { RateLimitManager } from "./RateLimitManager.js";
-
-/**
- * Service responsible for file handling during import.
- * Preserves all existing file handling capabilities including URL support.
- * Extracted from DataLoader to provide focused, testable file operations.
- */
-export class FileHandlerService {
-  private appwriteFolderPath: string;
-  private config: AppwriteConfig;
-  private importDataActions: ImportDataActions;
-  private rateLimitManager: RateLimitManager;
-
-  constructor(
-    appwriteFolderPath: string,
-    config: AppwriteConfig,
-    importDataActions: ImportDataActions,
-    rateLimitManager?: RateLimitManager
-  ) {
-    this.appwriteFolderPath = appwriteFolderPath;
-    this.config = config;
-    this.importDataActions = importDataActions;
-    this.rateLimitManager = rateLimitManager || new RateLimitManager();
-  }
-
-  /**
-   * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-   * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-   * and update the field with the file's ID if necessary.
-   * 
-   * Preserves existing file handling logic from DataLoader.
-   *
-   * @param attributeMappings - The attribute mappings from the import definition.
-   * @param context - The context object containing information about the database, collection, and document.
-   * @param item - The item being imported, used for resolving template paths in fileData mappings.
-   * @returns The attribute mappings updated with any necessary post-import actions.
-   */
-  getAttributeMappingsWithFileActions(
-    attributeMappings: AttributeMappings,
-    context: any,
-    item: any
-  ): AttributeMappings {
-    // Iterate over each attribute mapping to check for fileData attributes
-    return attributeMappings.map((mapping) => {
-      if (mapping.fileData) {
-        // Resolve the file path using the provided template, context, and item
-        let mappingFilePath = this.importDataActions.resolveTemplate(
-          mapping.fileData.path,
-          context,
-          item
-        );
-        
-        // Ensure the file path is absolute if it doesn't start with "http"
-        if (!mappingFilePath.toLowerCase().startsWith("http")) {
-          mappingFilePath = this.resolveLocalFilePath(mappingFilePath);
-        }
-        
-        // Define the after-import action to create a file and update the field
-        const afterImportAction = {
-          action: "createFileAndUpdateField",
-          params: [
-            "{dbId}",
-            "{collId}",
-            "{docId}",
-            mapping.targetKey,
-            `${this.config!.documentBucketId}_${context.dbName
-              .toLowerCase()
-              .replace(" ", "")}`, // Bucket ID pattern
-            mappingFilePath,
-            mapping.fileData.name,
-          ],
-        };
-        
-        // Add the after-import action to the mapping's postImportActions array
-        const postImportActions = mapping.postImportActions
-          ? [...mapping.postImportActions, afterImportAction]
-          : [afterImportAction];
-        
-        return { ...mapping, postImportActions };
-      }
-      
-      // Return the mapping unchanged if no fileData attribute is found
-      return mapping;
-    });
-  }
-
-  /**
-   * Resolves local file path, searching in subdirectories if needed.
-   * Preserves existing file search logic from DataLoader.
-   * 
-   * @param mappingFilePath - The relative file path from the mapping
-   * @returns Resolved absolute file path
-   */
-  private resolveLocalFilePath(mappingFilePath: string): string {
-    // First try the direct path
-    let fullPath = path.resolve(this.appwriteFolderPath, mappingFilePath);
-
-    // If file doesn't exist, search in subdirectories
-    if (!fs.existsSync(fullPath)) {
-      const findFileInDir = (dir: string): string | null => {
-        try {
-          const files = fs.readdirSync(dir);
-
-          for (const file of files) {
-            const filePath = path.join(dir, file);
-            const stat = fs.statSync(filePath);
-
-            if (stat.isDirectory()) {
-              // Recursively search subdirectories
-              const found = findFileInDir(filePath);
-              if (found) return found;
-            } else if (file === path.basename(mappingFilePath)) {
-              return filePath;
-            }
-          }
-        } catch (error) {
-          logger.warn(`Error reading directory ${dir}: ${error}`);
-        }
-        return null;
-      };
-
-      const foundPath = findFileInDir(this.appwriteFolderPath);
-      if (foundPath) {
-        mappingFilePath = foundPath;
-      } else {
-        logger.warn(
-          `File not found in any subdirectory: ${mappingFilePath}`
-        );
-        // Keep the original resolved path as fallback
-        mappingFilePath = fullPath;
-      }
-    } else {
-      mappingFilePath = fullPath;
-    }
-
-    return mappingFilePath;
-  }
-
-  /**
-   * Executes file-related post-import actions with rate limiting.
-   * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
-   * 
-   * @param context - The context containing document and collection information
-   * @param postImportActions - The post-import actions to execute
-   */
-  async executeFileActions(
-    context: any,
-    postImportActions: any[]
-  ): Promise<void> {
-    const fileActions = postImportActions.filter(
-      action => action.action === "createFileAndUpdateField"
-    );
-
-    if (fileActions.length === 0) {
-      return;
-    }
-
-    // Execute file actions with rate limiting
-    const filePromises = fileActions.map(action =>
-      this.rateLimitManager.fileUpload(() => this.executeFileAction(action, context))
-    );
-
-    try {
-      await Promise.allSettled(filePromises);
-    } catch (error) {
-      logger.error(`Error executing file actions for context: ${JSON.stringify(context, null, 2)}`, error);
-    }
-  }
-
-  /**
-   * Executes a single file action with proper error handling.
-   * 
-   * @param action - The file action to execute
-   * @param context - The context for template resolution
-   */
-  private async executeFileAction(action: any, context: any): Promise<void> {
-    try {
-      // Resolve parameters using existing template resolution
-      const resolvedParams = action.params.map((param: any) =>
-        this.importDataActions.resolveTemplate(param, context, context)
-      );
-
-      // Execute the createFileAndUpdateField action
-      await this.importDataActions.executeAction(
-        action.action,
-        resolvedParams,
-        context,
-        context
-      );
-    } catch (error) {
-      logger.error(
-        `Failed to execute file action '${action.action}' with params: ${JSON.stringify(action.params, null, 2)}`,
-        error
-      );
-      // Don't throw - we want to continue with other file operations
-    }
-  }
-
-  /**
-   * Validates that file paths exist before import begins.
-   * Provides early validation to catch file issues before processing starts.
-   * 
-   * @param attributeMappings - The attribute mappings to validate
-   * @param context - The context for template resolution
-   * @param item - The item for template resolution
-   * @returns Array of validation errors (empty if all valid)
-   */
-  validateFilePaths(
-    attributeMappings: AttributeMappings,
-    context: any,
-    item: any
-  ): string[] {
-    const errors: string[] = [];
-
-    for (const mapping of attributeMappings) {
-      if (mapping.fileData) {
-        try {
-          let filePath = this.importDataActions.resolveTemplate(
-            mapping.fileData.path,
-            context,
-            item
-          );
-
-          // Skip URL validation (URLs are handled by the action itself)
-          if (filePath.toLowerCase().startsWith("http")) {
-            continue;
-          }
-
-          // Check if local file exists
-          const resolvedPath = this.resolveLocalFilePath(filePath);
-          if (!fs.existsSync(resolvedPath)) {
-            errors.push(
-              `File not found for mapping '${mapping.targetKey}': ${resolvedPath}`
-            );
-          }
-        } catch (error) {
-          errors.push(
-            `Error validating file path for mapping '${mapping.targetKey}': ${error}`
-          );
-        }
-      }
-    }
-
-    return errors;
-  }
-
-  /**
-   * Gets file statistics for import planning.
-   * Helps estimate import time and resource requirements.
-   * 
-   * @param attributeMappings - The attribute mappings to analyze
-   * @param items - The items that will be imported
-   * @returns File statistics object
-   */
-  getFileStatistics(attributeMappings: AttributeMappings, items: any[]): {
-    totalFiles: number;
-    urlFiles: number;
-    localFiles: number;
-    estimatedSize: number;
-  } {
-    let totalFiles = 0;
-    let urlFiles = 0;
-    let localFiles = 0;
-    let estimatedSize = 0;
-
-    const fileAttributeMappings = attributeMappings.filter(mapping => mapping.fileData);
-    
-    for (const item of items) {
-      for (const mapping of fileAttributeMappings) {
-        totalFiles++;
-        
-        try {
-          const filePath = this.importDataActions.resolveTemplate(
-            mapping.fileData!.path,
-            {},
-            item
-          );
-
-          if (filePath.toLowerCase().startsWith("http")) {
-            urlFiles++;
-          } else {
-            localFiles++;
-            try {
-              const resolvedPath = this.resolveLocalFilePath(filePath);
-              if (fs.existsSync(resolvedPath)) {
-                const stats = fs.statSync(resolvedPath);
-                estimatedSize += stats.size;
-              }
-            } catch (error) {
-              // Ignore errors for statistics
-            }
-          }
-        } catch (error) {
-          // Ignore errors for statistics
-        }
-      }
-    }
-
-    return {
-      totalFiles,
-      urlFiles,
-      localFiles,
-      estimatedSize
-    };
-  }
-}