@njdamstra/appwrite-utils-cli 1.8.9

package/dist/migrations/services/UserMappingService.js

@@ -0,0 +1,277 @@
+import { z } from "zod";
+import { ID } from "node-appwrite";
+import { AuthUserCreateSchema } from "../../schemas/authUser.js";
+import { logger } from "../../shared/logging.js";
+/**
+ * Service responsible for user mapping and deduplication during import.
+ * Preserves all existing sophisticated user handling logic from DataLoader.
+ * Extracted to provide focused, testable user management operations.
+ */
+export class UserMappingService {
+    config;
+    dataTransformationService;
+    // Maps to hold email and phone to user ID mappings for uniqueness in User Accounts
+    emailToUserIdMap = new Map();
+    phoneToUserIdMap = new Map();
+    userIdSet = new Set();
+    // Map to hold the merged user map for relationship resolution
+    // Will hold an array of the old user ID's that are mapped to the same new user ID
+    // For example, if there are two users with the same email, they will both be mapped to the same new user ID
+    // Prevents duplicate users with the other two maps below it and allows me to keep the old ID's
+    mergedUserMap = new Map();
+    // Map to track existing users
+    userExistsMap = new Map();
+    constructor(config, dataTransformationService) {
+        this.config = config;
+        this.dataTransformationService = dataTransformationService;
+    }
+    /**
+     * Initializes user maps with existing users from the system.
+     * Preserves existing user loading logic from DataLoader.
+     *
+     * @param existingUsers - Array of existing users from the system
+     */
+    initializeWithExistingUsers(existingUsers) {
+        // Clear existing maps
+        this.emailToUserIdMap.clear();
+        this.phoneToUserIdMap.clear();
+        this.userIdSet.clear();
+        this.userExistsMap.clear();
+        // Iterate over the users and setup our maps ahead of time for email and phone
+        for (const user of existingUsers) {
+            if (user.email) {
+                this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
+            }
+            if (user.phone) {
+                this.phoneToUserIdMap.set(user.phone, user.$id);
+            }
+            this.userExistsMap.set(user.$id, true);
+            this.userIdSet.add(user.$id);
+        }
+        logger.info(`Initialized user maps with ${existingUsers.length} existing users`);
+    }
+    /**
+     * Generates a unique ID that doesn't conflict with existing user IDs.
+     * Preserves existing unique ID generation logic from DataLoader.
+     *
+     * @param collectionName - The collection name for context
+     * @returns A truly unique ID
+     */
+    getTrueUniqueUserId(collectionName) {
+        let newId = ID.unique();
+        let condition = this.userExistsMap.has(newId) ||
+            this.userIdSet.has(newId) ||
+            Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+            Array.from(this.phoneToUserIdMap.values()).includes(newId);
+        while (condition) {
+            newId = ID.unique();
+            condition =
+                this.userExistsMap.has(newId) ||
+                    this.userIdSet.has(newId) ||
+                    Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+                    Array.from(this.phoneToUserIdMap.values()).includes(newId);
+        }
+        return newId;
+    }
+    /**
+     * Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
+     * and then returning the transformed item without user-specific keys.
+     *
+     * Preserves existing sophisticated user deduplication logic from DataLoader.
+     *
+     * @param item - The raw item to be processed.
+     * @param attributeMappings - The attribute mappings for the item.
+     * @param primaryKeyField - The primary key field name
+     * @param newId - The proposed new ID for the user
+     * @returns Object containing transformed item, existing ID if duplicate, and user data
+     */
+    prepareUserData(item, attributeMappings, primaryKeyField, newId) {
+        // Ensure we have a truly unique ID
+        if (this.userIdSet.has(newId) ||
+            this.userExistsMap.has(newId) ||
+            Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+            Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
+            newId = this.getTrueUniqueUserId("users");
+        }
+        let transformedItem = this.dataTransformationService.transformData(item, attributeMappings);
+        let userData = AuthUserCreateSchema.safeParse(transformedItem);
+        if (userData.data?.email) {
+            userData.data.email = userData.data.email.toLowerCase();
+        }
+        if (!userData.success || !(userData.data.email || userData.data.phone)) {
+            logger.error(`Invalid user data: ${JSON.stringify(userData.error?.issues, undefined, 2)} or missing email/phone`);
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem,
+                existingId: undefined,
+                userData: {
+                    rawData: item,
+                    finalData: transformedItem,
+                },
+            };
+        }
+        const email = userData.data.email?.toLowerCase();
+        const phone = userData.data.phone;
+        let existingId;
+        // Check for duplicate email and phone
+        if (email && this.emailToUserIdMap.has(email)) {
+            existingId = this.emailToUserIdMap.get(email);
+            if (phone && !this.phoneToUserIdMap.has(phone)) {
+                this.phoneToUserIdMap.set(phone, newId);
+            }
+        }
+        else if (phone && this.phoneToUserIdMap.has(phone)) {
+            existingId = this.phoneToUserIdMap.get(phone);
+            if (email && !this.emailToUserIdMap.has(email)) {
+                this.emailToUserIdMap.set(email, newId);
+            }
+        }
+        else {
+            if (email)
+                this.emailToUserIdMap.set(email, newId);
+            if (phone)
+                this.phoneToUserIdMap.set(phone, newId);
+        }
+        if (existingId) {
+            userData.data.userId = existingId;
+            const mergedUsers = this.mergedUserMap.get(existingId) || [];
+            mergedUsers.push(`${item[primaryKeyField]}`);
+            this.mergedUserMap.set(existingId, mergedUsers);
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem: {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                },
+                existingId,
+                userData: {
+                    rawData: item,
+                    finalData: userData.data,
+                },
+            };
+        }
+        else {
+            existingId = newId;
+            userData.data.userId = existingId;
+        }
+        const userKeys = ["email", "phone", "name", "labels", "prefs"];
+        userKeys.forEach((key) => {
+            if (transformedItem.hasOwnProperty(key)) {
+                delete transformedItem[key];
+            }
+        });
+        this.userIdSet.add(existingId);
+        return {
+            transformedItem: {
+                ...transformedItem,
+                userId: existingId,
+                docId: existingId,
+            },
+            existingId,
+            userData: {
+                rawData: item,
+                finalData: userData.data,
+            },
+        };
+    }
+    /**
+     * Checks if a collection is the users collection based on configuration.
+     *
+     * @param collectionName - The collection name to check
+     * @returns True if this is the users collection
+     */
+    isUsersCollection(collectionName) {
+        return !!(this.config.usersCollectionName &&
+            this.getCollectionKey(this.config.usersCollectionName) ===
+                this.getCollectionKey(collectionName));
+    }
+    /**
+     * Helper method to generate a consistent key for collections.
+     * Preserves existing logic from DataLoader.
+     */
+    getCollectionKey(name) {
+        return name.toLowerCase().replace(" ", "");
+    }
+    /**
+     * Gets merged user mappings for relationship resolution.
+     *
+     * @returns Map of merged user IDs to arrays of original IDs
+     */
+    getMergedUserMap() {
+        return new Map(this.mergedUserMap);
+    }
+    /**
+     * Finds the new user ID for an old user ID, considering merged users.
+     * Preserves existing merged user lookup logic from DataLoader.
+     *
+     * @param oldId - The old user ID to look up
+     * @returns The new user ID if found, otherwise the original ID
+     */
+    findNewUserIdForOldId(oldId) {
+        // Check if this old ID was merged into another user
+        for (const [newUserId, oldUserIds] of this.mergedUserMap.entries()) {
+            if (oldUserIds.includes(`${oldId}`)) {
+                return newUserId;
+            }
+        }
+        // If not found in merged users, return the original ID
+        return oldId;
+    }
+    /**
+     * Merges user data from duplicate entries.
+     * Preserves existing user data merging logic.
+     *
+     * @param existingUserData - The existing user data
+     * @param newUserData - The new user data to merge
+     * @returns Merged user data
+     */
+    mergeUserData(existingUserData, newUserData) {
+        return this.dataTransformationService.mergeObjects(existingUserData, newUserData);
+    }
+    /**
+     * Gets statistics about user mapping operations.
+     * Useful for import planning and reporting.
+     *
+     * @returns User mapping statistics
+     */
+    getStatistics() {
+        let totalMergedOldIds = 0;
+        for (const oldIds of this.mergedUserMap.values()) {
+            totalMergedOldIds += oldIds.length;
+        }
+        return {
+            totalExistingUsers: this.userExistsMap.size,
+            emailMappings: this.emailToUserIdMap.size,
+            phoneMappings: this.phoneToUserIdMap.size,
+            mergedUsers: this.mergedUserMap.size,
+            totalMergedOldIds,
+        };
+    }
+    /**
+     * Validates user mapping configuration.
+     * Ensures the user mapping setup is correct before import.
+     *
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateConfiguration() {
+        const errors = [];
+        if (!this.config.usersCollectionName) {
+            errors.push("usersCollectionName is not configured");
+        }
+        if (!this.config.documentBucketId) {
+            errors.push("documentBucketId is not configured");
+        }
+        return errors;
+    }
+}

package/dist/migrations/services/ValidationService.d.ts

@@ -0,0 +1,74 @@
+import type { AttributeMappings, ImportDef, CollectionCreate } from "@njdamstra/appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+/**
+ * Service responsible for validation during import operations.
+ * Provides centralized validation logic extracted from ImportDataActions and DataLoader.
+ */
+export declare class ValidationService {
+    private importDataActions;
+    constructor(importDataActions: ImportDataActions);
+    /**
+     * Validates a single data item based on defined validation rules.
+     * Preserves existing validation logic from ImportDataActions.
+     *
+     * @param item - The data item to validate
+     * @param attributeMappings - The attribute mappings for the data item
+     * @param context - The context for resolving templated parameters in validation rules
+     * @returns True if the item is valid, false otherwise
+     */
+    validateDataItem(item: any, attributeMappings: AttributeMappings, context: {
+        [key: string]: any;
+    }): boolean;
+    /**
+     * Validates an import definition before processing begins.
+     * Provides early validation to catch configuration issues.
+     *
+     * @param importDef - The import definition to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateImportDefinition(importDef: ImportDef): string[];
+    /**
+     * Validates a collection configuration for import compatibility.
+     * Ensures the collection has the necessary attributes for import operations.
+     *
+     * @param collection - The collection to validate
+     * @param importDefs - The import definitions that will be used with this collection
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateCollectionForImport(collection: CollectionCreate, importDefs: ImportDef[]): string[];
+    /**
+     * Validates data consistency across multiple collections.
+     * Checks for relationship integrity and data consistency.
+     *
+     * @param collections - Array of collections to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateCrossCollectionConsistency(collections: CollectionCreate[]): string[];
+    /**
+     * Performs comprehensive pre-import validation.
+     * Validates all aspects of the import configuration before starting.
+     *
+     * @param collections - Collections that will be imported
+     * @param appwriteFolderPath - Path to the appwrite folder for file validation
+     * @returns Validation result with errors and warnings
+     */
+    performPreImportValidation(collections: CollectionCreate[], appwriteFolderPath: string): {
+        errors: string[];
+        warnings: string[];
+        isValid: boolean;
+    };
+    /**
+     * Validates import progress and data integrity during import.
+     * Can be used for periodic validation during long-running imports.
+     *
+     * @param processedItems - Number of items processed so far
+     * @param totalItems - Total number of items to process
+     * @param errorCount - Number of errors encountered
+     * @returns Validation status with recommendations
+     */
+    validateImportProgress(processedItems: number, totalItems: number, errorCount: number): {
+        status: "healthy" | "warning" | "critical";
+        message: string;
+        shouldContinue: boolean;
+    };
+}

package/dist/migrations/services/ValidationService.js

@@ -0,0 +1,260 @@
+import { logger } from "../../shared/logging.js";
+import fs from "fs";
+import path from "path";
+/**
+ * Service responsible for validation during import operations.
+ * Provides centralized validation logic extracted from ImportDataActions and DataLoader.
+ */
+export class ValidationService {
+    importDataActions;
+    constructor(importDataActions) {
+        this.importDataActions = importDataActions;
+    }
+    /**
+     * Validates a single data item based on defined validation rules.
+     * Preserves existing validation logic from ImportDataActions.
+     *
+     * @param item - The data item to validate
+     * @param attributeMappings - The attribute mappings for the data item
+     * @param context - The context for resolving templated parameters in validation rules
+     * @returns True if the item is valid, false otherwise
+     */
+    validateDataItem(item, attributeMappings, context) {
+        try {
+            return this.importDataActions.validateItem(item, attributeMappings, context);
+        }
+        catch (error) {
+            logger.error(`Validation error for item: ${JSON.stringify(item, null, 2)}`, error);
+            return false;
+        }
+    }
+    /**
+     * Validates an import definition before processing begins.
+     * Provides early validation to catch configuration issues.
+     *
+     * @param importDef - The import definition to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateImportDefinition(importDef) {
+        const errors = [];
+        // Validate required fields
+        if (!importDef.filePath) {
+            errors.push("filePath is required");
+        }
+        if (!importDef.attributeMappings || importDef.attributeMappings.length === 0) {
+            errors.push("attributeMappings are required");
+        }
+        // Validate attribute mappings
+        if (importDef.attributeMappings) {
+            for (let i = 0; i < importDef.attributeMappings.length; i++) {
+                const mapping = importDef.attributeMappings[i];
+                if (!mapping.targetKey) {
+                    errors.push(`attributeMappings[${i}]: targetKey is required`);
+                }
+                if (!mapping.oldKey && !mapping.oldKeys && mapping.valueToSet === undefined) {
+                    errors.push(`attributeMappings[${i}]: must have either oldKey, oldKeys, or valueToSet`);
+                }
+                // Validate file data if present
+                if (mapping.fileData) {
+                    if (!mapping.fileData.path) {
+                        errors.push(`attributeMappings[${i}].fileData: path is required`);
+                    }
+                    if (!mapping.fileData.name) {
+                        errors.push(`attributeMappings[${i}].fileData: name is required`);
+                    }
+                }
+                // Validate converters if present
+                if (mapping.converters && !Array.isArray(mapping.converters)) {
+                    errors.push(`attributeMappings[${i}].converters: must be an array`);
+                }
+                // Validate validation actions if present
+                if (mapping.validationActions) {
+                    if (!Array.isArray(mapping.validationActions)) {
+                        errors.push(`attributeMappings[${i}].validationActions: must be an array`);
+                    }
+                    else {
+                        for (let j = 0; j < mapping.validationActions.length; j++) {
+                            const action = mapping.validationActions[j];
+                            if (!action.action) {
+                                errors.push(`attributeMappings[${i}].validationActions[${j}]: action is required`);
+                            }
+                            if (!Array.isArray(action.params)) {
+                                errors.push(`attributeMappings[${i}].validationActions[${j}]: params must be an array`);
+                            }
+                        }
+                    }
+                }
+                // Validate post-import actions if present
+                if (mapping.postImportActions) {
+                    if (!Array.isArray(mapping.postImportActions)) {
+                        errors.push(`attributeMappings[${i}].postImportActions: must be an array`);
+                    }
+                    else {
+                        for (let j = 0; j < mapping.postImportActions.length; j++) {
+                            const action = mapping.postImportActions[j];
+                            if (!action.action) {
+                                errors.push(`attributeMappings[${i}].postImportActions[${j}]: action is required`);
+                            }
+                            if (!Array.isArray(action.params)) {
+                                errors.push(`attributeMappings[${i}].postImportActions[${j}]: params must be an array`);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Validate ID mappings if present
+        if (importDef.idMappings) {
+            for (let i = 0; i < importDef.idMappings.length; i++) {
+                const idMapping = importDef.idMappings[i];
+                if (!idMapping.sourceField) {
+                    errors.push(`idMappings[${i}]: sourceField is required`);
+                }
+                if (!idMapping.targetField) {
+                    errors.push(`idMappings[${i}]: targetField is required`);
+                }
+                if (!idMapping.targetCollection) {
+                    errors.push(`idMappings[${i}]: targetCollection is required`);
+                }
+            }
+        }
+        // Validate update mapping if present
+        if (importDef.updateMapping) {
+            if (!importDef.updateMapping.originalIdField) {
+                errors.push("updateMapping: originalIdField is required");
+            }
+            if (!importDef.updateMapping.targetField) {
+                errors.push("updateMapping: targetField is required");
+            }
+        }
+        return errors;
+    }
+    /**
+     * Validates a collection configuration for import compatibility.
+     * Ensures the collection has the necessary attributes for import operations.
+     *
+     * @param collection - The collection to validate
+     * @param importDefs - The import definitions that will be used with this collection
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateCollectionForImport(collection, importDefs) {
+        const errors = [];
+        if (!collection.name) {
+            errors.push("Collection name is required");
+        }
+        if (!collection.attributes || collection.attributes.length === 0) {
+            errors.push("Collection must have attributes defined");
+        }
+        // Validate that target keys in import definitions exist as attributes
+        for (const importDef of importDefs) {
+            for (const mapping of importDef.attributeMappings) {
+                const attributeExists = collection.attributes?.some(attr => attr.key === mapping.targetKey);
+                if (!attributeExists) {
+                    errors.push(`Attribute '${mapping.targetKey}' referenced in import mapping not found in collection`);
+                }
+            }
+        }
+        return errors;
+    }
+    /**
+     * Validates data consistency across multiple collections.
+     * Checks for relationship integrity and data consistency.
+     *
+     * @param collections - Array of collections to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validateCrossCollectionConsistency(collections) {
+        const errors = [];
+        const collectionNames = new Set(collections.map(c => c.name));
+        for (const collection of collections) {
+            if (!collection.importDefs)
+                continue;
+            for (const importDef of collection.importDefs) {
+                if (!importDef.idMappings)
+                    continue;
+                for (const idMapping of importDef.idMappings) {
+                    // Check if target collection exists
+                    if (!collectionNames.has(idMapping.targetCollection)) {
+                        errors.push(`Collection '${collection.name}' references non-existent target collection '${idMapping.targetCollection}' in ID mapping`);
+                    }
+                }
+            }
+        }
+        return errors;
+    }
+    /**
+     * Performs comprehensive pre-import validation.
+     * Validates all aspects of the import configuration before starting.
+     *
+     * @param collections - Collections that will be imported
+     * @param appwriteFolderPath - Path to the appwrite folder for file validation
+     * @returns Validation result with errors and warnings
+     */
+    performPreImportValidation(collections, appwriteFolderPath) {
+        const errors = [];
+        const warnings = [];
+        // Validate each collection
+        for (const collection of collections) {
+            const collectionErrors = this.validateCollectionForImport(collection, collection.importDefs || []);
+            errors.push(...collectionErrors.map(err => `${collection.name}: ${err}`));
+            // Validate each import definition
+            if (collection.importDefs) {
+                for (let i = 0; i < collection.importDefs.length; i++) {
+                    const importDef = collection.importDefs[i];
+                    const importDefErrors = this.validateImportDefinition(importDef);
+                    errors.push(...importDefErrors.map(err => `${collection.name}.importDefs[${i}]: ${err}`));
+                    // Check if import file exists
+                    try {
+                        const filePath = path.resolve(appwriteFolderPath, importDef.filePath);
+                        if (!fs.existsSync(filePath)) {
+                            errors.push(`${collection.name}.importDefs[${i}]: Import file not found: ${filePath}`);
+                        }
+                    }
+                    catch (error) {
+                        warnings.push(`${collection.name}.importDefs[${i}]: Could not validate file existence: ${error}`);
+                    }
+                }
+            }
+        }
+        // Validate cross-collection consistency
+        const crossCollectionErrors = this.validateCrossCollectionConsistency(collections);
+        errors.push(...crossCollectionErrors);
+        return {
+            errors,
+            warnings,
+            isValid: errors.length === 0,
+        };
+    }
+    /**
+     * Validates import progress and data integrity during import.
+     * Can be used for periodic validation during long-running imports.
+     *
+     * @param processedItems - Number of items processed so far
+     * @param totalItems - Total number of items to process
+     * @param errorCount - Number of errors encountered
+     * @returns Validation status with recommendations
+     */
+    validateImportProgress(processedItems, totalItems, errorCount) {
+        const errorRate = errorCount / Math.max(processedItems, 1);
+        const progress = processedItems / totalItems;
+        if (errorRate > 0.5) {
+            return {
+                status: "critical",
+                message: `Error rate too high: ${(errorRate * 100).toFixed(1)}%. Consider stopping import.`,
+                shouldContinue: false,
+            };
+        }
+        if (errorRate > 0.1) {
+            return {
+                status: "warning",
+                message: `Elevated error rate: ${(errorRate * 100).toFixed(1)}%. Monitor closely.`,
+                shouldContinue: true,
+            };
+        }
+        return {
+            status: "healthy",
+            message: `Import progressing normally: ${(progress * 100).toFixed(1)}% complete, ${(errorRate * 100).toFixed(1)}% error rate.`,
+            shouldContinue: true,
+        };
+    }
+}

package/dist/migrations/transfer.d.ts

@@ -0,0 +1,26 @@
+import { Databases, Storage, Users, type Models } from "node-appwrite";
+export interface TransferOptions {
+    fromDb: Models.Database | undefined;
+    targetDb: Models.Database | undefined;
+    isRemote: boolean;
+    collections?: string[];
+    transferEndpoint?: string;
+    transferProject?: string;
+    transferKey?: string;
+    sourceBucket?: Models.Bucket;
+    targetBucket?: Models.Bucket;
+    transferUsers?: boolean;
+}
+export declare const transferStorageLocalToLocal: (storage: Storage, fromBucketId: string, toBucketId: string) => Promise<void>;
+export declare const transferStorageLocalToRemote: (localStorage: Storage, endpoint: string, projectId: string, apiKey: string, fromBucketId: string, toBucketId: string) => Promise<void>;
+/**
+ * Transfers all collections and documents from one local database to another local database.
+ *
+ * @param {Databases} localDb - The local database instance.
+ * @param {string} fromDbId - The ID of the source database.
+ * @param {string} targetDbId - The ID of the target database.
+ * @return {Promise<void>} A promise that resolves when the transfer is complete.
+ */
+export declare const transferDatabaseLocalToLocal: (localDb: Databases, fromDbId: string, targetDbId: string) => Promise<void>;
+export declare const transferDatabaseLocalToRemote: (localDb: Databases, endpoint: string, projectId: string, apiKey: string, fromDbId: string, toDbId: string) => Promise<void>;
+export declare const transferUsersLocalToRemote: (localUsers: Users, endpoint: string, projectId: string, apiKey: string) => Promise<void>;