@njdamstra/appwrite-utils-cli 1.8.9

package/dist/migrations/relationships.js

@@ -0,0 +1,204 @@
+import { Databases, Query } from "node-appwrite";
+import { fetchAllCollections } from "../collections/methods.js";
+import { logger } from "../shared/logging.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+/**
+ * Finds collections that have defined relationship attributes.
+ */
+export const findCollectionsWithRelationships = (config) => {
+    const toReturn = new Map();
+    if (!config.collections) {
+        return toReturn;
+    }
+    for (const collection of config.collections) {
+        if (collection.attributes) {
+            for (const attribute of collection.attributes) {
+                if (attribute.type === "relationship" &&
+                    attribute.twoWay &&
+                    attribute.side === "parent") {
+                    toReturn.set(collection.name, toReturn.get(collection.name) || []);
+                    toReturn
+                        .get(collection.name)
+                        ?.push(attribute);
+                }
+            }
+        }
+    }
+    return toReturn;
+};
+export async function resolveAndUpdateRelationships(dbId, database, config) {
+    const collections = await fetchAllCollections(dbId, database);
+    const collectionsWithRelationships = findCollectionsWithRelationships(config);
+    // Process each collection sequentially
+    for (const collection of collections) {
+        MessageFormatter.processing(`Processing collection: ${collection.name} (${collection.$id})`, { prefix: "Migration" });
+        const relAttributeMap = collectionsWithRelationships.get(collection.name); // Get the relationship attributes for the collections
+        if (!relAttributeMap) {
+            MessageFormatter.info(`No mapping found for collection: ${collection.name}, skipping...`, { prefix: "Migration" });
+            continue;
+        }
+        await processCollection(dbId, database, collection, relAttributeMap);
+    }
+    MessageFormatter.success(`Completed relationship resolution and update for database ID: ${dbId}`, { prefix: "Migration" });
+}
+async function processCollection(dbId, database, collection, relAttributeMap) {
+    let after; // For pagination
+    let hasMore = true;
+    while (hasMore) {
+        const response = await database.listDocuments(dbId, collection.$id, [
+            Query.limit(100), // Fetch documents in batches of 100
+            ...(after ? [Query.cursorAfter(after)] : []),
+        ]);
+        const documents = response.documents;
+        MessageFormatter.info(`Fetched ${documents.length} documents from collection: ${collection.name}`, { prefix: "Migration" });
+        if (documents.length > 0) {
+            const updates = await prepareDocumentUpdates(database, dbId, collection.name, documents, relAttributeMap);
+            // Execute updates for the current batch
+            await executeUpdatesInBatches(dbId, database, updates);
+        }
+        if (documents.length === 100) {
+            after = documents[documents.length - 1].$id; // Prepare for the next page
+        }
+        else {
+            hasMore = false; // No more documents to fetch
+        }
+    }
+}
+async function findDocumentsByOriginalId(database, dbId, targetCollection, targetKey, originalId) {
+    const relatedCollectionId = targetCollection.$id;
+    const collection = await database.listCollections(dbId, [
+        Query.equal("$id", relatedCollectionId),
+    ]);
+    if (collection.total === 0) {
+        MessageFormatter.warning(`Collection ${relatedCollectionId} doesn't exist, skipping...`, { prefix: "Migration" });
+        return undefined;
+    }
+    const targetAttr = collection.collections[0].attributes.find(
+    // @ts-ignore
+    (attr) => attr.key === targetKey);
+    if (!targetAttr) {
+        MessageFormatter.warning(`Attribute ${targetKey} not found in collection ${relatedCollectionId}, skipping...`, { prefix: "Migration" });
+        return undefined;
+    }
+    let queries = [];
+    if (targetAttr.array) {
+        // @ts-ignore
+        queries.push(Query.contains(targetKey, originalId));
+    }
+    else {
+        queries.push(Query.equal(targetKey, originalId));
+    }
+    const response = await database.listDocuments(dbId, relatedCollectionId, [
+        ...queries,
+        Query.limit(500), // Adjust the limit based on your needs or implement pagination
+    ]);
+    if (response.documents.length < 0) {
+        return undefined;
+    }
+    else if (response.documents.length > 0) {
+        return response.documents;
+    }
+    else {
+        return undefined;
+    }
+}
+async function prepareDocumentUpdates(database, dbId, collectionName, documents, relationships) {
+    MessageFormatter.processing(`Preparing updates for collection: ${collectionName}`, { prefix: "Migration" });
+    const updates = [];
+    const thisCollection = (await database.listCollections(dbId, [Query.equal("name", collectionName)])).collections[0];
+    const thisCollectionId = thisCollection?.$id;
+    if (!thisCollectionId) {
+        MessageFormatter.warning(`No collection found with name: ${collectionName}`, { prefix: "Migration" });
+        return [];
+    }
+    for (const doc of documents) {
+        let updatePayload = {};
+        for (const rel of relationships) {
+            // Skip if not dealing with the parent side of a two-way relationship
+            if (rel.twoWay && rel.side !== "parent") {
+                MessageFormatter.info("Skipping non-parent side of two-way relationship...", { prefix: "Migration" });
+                continue;
+            }
+            const isSingleReference = rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
+            const originalIdField = rel.importMapping?.originalIdField;
+            const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
+            if (!originalIdField) {
+                MessageFormatter.warning("Missing originalIdField in importMapping, skipping...", { prefix: "Migration" });
+                continue;
+            }
+            const originalId = doc[originalIdField];
+            if (!originalId) {
+                continue;
+            }
+            const relatedCollection = (await database.listCollections(dbId, [
+                Query.equal("name", rel.relatedCollection),
+            ])).collections[0];
+            if (!relatedCollection) {
+                MessageFormatter.warning(`Related collection ${rel.relatedCollection} not found, skipping...`, { prefix: "Migration" });
+                continue;
+            }
+            const foundDocuments = await findDocumentsByOriginalId(database, dbId, relatedCollection, targetField, String(originalId));
+            if (foundDocuments && foundDocuments.length > 0) {
+                const relationshipKey = rel.key;
+                const existingRefs = doc[relationshipKey] || [];
+                let existingRefIds = [];
+                if (Array.isArray(existingRefs)) {
+                    // @ts-ignore
+                    existingRefIds = existingRefs.map((ref) => ref.$id);
+                }
+                else if (existingRefs) {
+                    // @ts-ignore
+                    existingRefIds = [existingRefs.$id];
+                }
+                const newRefs = foundDocuments.map((fd) => fd.$id);
+                const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
+                // Update logic based on the relationship cardinality
+                updatePayload[relationshipKey] = isSingleReference
+                    ? newRefs[0] || existingRefIds[0]
+                    : allRefs;
+                MessageFormatter.info(`Updating ${relationshipKey} with ${allRefs.length} refs`, { prefix: "Migration" });
+            }
+        }
+        if (Object.keys(updatePayload).length > 0) {
+            updates.push({
+                collectionId: thisCollectionId,
+                documentId: doc.$id,
+                updatePayload: updatePayload,
+            });
+        }
+    }
+    return updates;
+}
+async function processInBatches(items, batchSize, processFunction) {
+    const maxParallelBatches = 25; // Adjust this value to control the number of parallel batches
+    let currentIndex = 0;
+    let activeBatchPromises = [];
+    while (currentIndex < items.length) {
+        // While there's still data to process and we haven't reached our parallel limit
+        while (currentIndex < items.length &&
+            activeBatchPromises.length < maxParallelBatches) {
+            const batch = items.slice(currentIndex, currentIndex + batchSize);
+            currentIndex += batchSize;
+            // Add new batch processing promise to the array
+            activeBatchPromises.push(processFunction(batch));
+        }
+        // Wait for one of the batch processes to complete
+        await Promise.race(activeBatchPromises).then(() => {
+            // Remove the resolved promise from the activeBatchPromises array
+            activeBatchPromises = activeBatchPromises.filter((p) => p !== Promise.race(activeBatchPromises));
+        });
+    }
+    // After processing all batches, ensure all active promises are resolved
+    await Promise.all(activeBatchPromises);
+}
+async function executeUpdatesInBatches(dbId, database, updates) {
+    const batchSize = 25; // Adjust based on your rate limit and performance testing
+    for (let i = 0; i < updates.length; i += batchSize) {
+        const batch = updates.slice(i, i + batchSize);
+        await Promise.all(batch.map((update) => database
+            .updateDocument(dbId, update.collectionId, update.documentId, update.updatePayload)
+            .catch((error) => {
+            logger.error(`Error updating doc ${update.documentId} in ${dbId}, update payload: ${JSON.stringify(update.updatePayload, undefined, 4)}, error: ${error}`);
+        })));
+    }
+}

package/dist/migrations/services/DataTransformationService.d.ts

@@ -0,0 +1,55 @@
+import type { AttributeMappings, ConfigDatabase, CollectionCreate } from "@njdamstra/appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+/**
+ * Service responsible for data transformation during import.
+ * Extracted from DataLoader to provide focused, testable transformation logic.
+ */
+export declare class DataTransformationService {
+    private importDataActions;
+    constructor(importDataActions: ImportDataActions);
+    /**
+     * Transforms the given item based on the provided attribute mappings.
+     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+     *
+     * Preserves existing transformation logic from DataLoader.
+     *
+     * @param item - The item to be transformed.
+     * @param attributeMappings - The mappings that define how each attribute should be transformed.
+     * @returns The transformed item.
+     */
+    transformData(item: any, attributeMappings: AttributeMappings): any;
+    /**
+     * Creates a context object for data transformation.
+     * Preserves existing context creation logic from DataLoader.
+     *
+     * @param db - The database configuration
+     * @param collection - The collection configuration
+     * @param item - The raw item data
+     * @param docId - The document ID
+     * @returns Context object for transformation
+     */
+    createContext(db: ConfigDatabase, collection: CollectionCreate, item: any, docId: string): any;
+    /**
+     * Merges two objects by updating the source object with the target object's values.
+     * Preserves existing merge logic from DataLoader.
+     *
+     * It iterates through the target object's keys and updates the source object if:
+     * - The source object has the key.
+     * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
+     *
+     * @param source - The source object to be updated.
+     * @param update - The target object with values to update the source object.
+     * @returns The updated source object.
+     */
+    mergeObjects(source: any, update: any): any;
+    /**
+     * Validates the transformed data item using existing validation logic.
+     *
+     * @param transformedData - The transformed data to validate
+     * @param attributeMappings - The attribute mappings containing validation rules
+     * @param context - The context for validation
+     * @returns True if valid, false otherwise
+     */
+    validateTransformedData(transformedData: any, attributeMappings: AttributeMappings, context: any): boolean;
+}

package/dist/migrations/services/DataTransformationService.js

@@ -0,0 +1,158 @@
+import { convertObjectByAttributeMappings } from "../../utils/dataConverters.js";
+import { logger } from "../../shared/logging.js";
+/**
+ * Service responsible for data transformation during import.
+ * Extracted from DataLoader to provide focused, testable transformation logic.
+ */
+export class DataTransformationService {
+    importDataActions;
+    constructor(importDataActions) {
+        this.importDataActions = importDataActions;
+    }
+    /**
+     * Transforms the given item based on the provided attribute mappings.
+     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+     *
+     * Preserves existing transformation logic from DataLoader.
+     *
+     * @param item - The item to be transformed.
+     * @param attributeMappings - The mappings that define how each attribute should be transformed.
+     * @returns The transformed item.
+     */
+    transformData(item, attributeMappings) {
+        try {
+            // Convert the item using the attribute mappings provided
+            const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
+            // Run additional converter functions on the converted item, if any
+            return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
+        }
+        catch (error) {
+            logger.error(`Error transforming data for item: ${JSON.stringify(item, null, 2)}`, error);
+            throw error;
+        }
+    }
+    /**
+     * Creates a context object for data transformation.
+     * Preserves existing context creation logic from DataLoader.
+     *
+     * @param db - The database configuration
+     * @param collection - The collection configuration
+     * @param item - The raw item data
+     * @param docId - The document ID
+     * @returns Context object for transformation
+     */
+    createContext(db, collection, item, docId) {
+        return {
+            ...item, // Spread the item data for easy access to its properties
+            dbId: db.$id,
+            dbName: db.name,
+            collId: collection.$id,
+            collName: collection.name,
+            docId: docId,
+            createdDoc: {}, // Initially empty, to be updated when the document is created
+        };
+    }
+    /**
+     * Merges two objects by updating the source object with the target object's values.
+     * Preserves existing merge logic from DataLoader.
+     *
+     * It iterates through the target object's keys and updates the source object if:
+     * - The source object has the key.
+     * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
+     *
+     * @param source - The source object to be updated.
+     * @param update - The target object with values to update the source object.
+     * @returns The updated source object.
+     */
+    mergeObjects(source, update) {
+        // Create a new object to hold the merged result
+        const result = { ...source };
+        // Loop through the keys of the object we care about
+        for (const [key, value] of Object.entries(source)) {
+            // Check if the key exists in the target object
+            if (!Object.hasOwn(update, key)) {
+                // If the key doesn't exist, we can just skip it
+                continue;
+            }
+            if (update[key] === value) {
+                continue;
+            }
+            // If the value ain't here, we can just do whatever man
+            if (value === undefined || value === null || value === "") {
+                // If the update key is defined
+                if (update[key] !== undefined &&
+                    update[key] !== null &&
+                    update[key] !== "") {
+                    // might as well use it eh?
+                    result[key] = update[key];
+                }
+                // ELSE if the value is an array, because it would then not be === to those things above
+            }
+            else if (Array.isArray(value)) {
+                // Get the update value
+                const updateValue = update[key];
+                // If the update value is an array, concatenate and remove duplicates
+                // and poopy data
+                if (Array.isArray(updateValue)) {
+                    result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+                else {
+                    // If the update value is not an array, just use it
+                    result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+            }
+            else if (typeof value === "object" && !Array.isArray(value)) {
+                // If the value is an object, we need to merge it
+                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
+                    result[key] = this.mergeObjects(value, update[key]);
+                }
+            }
+            else {
+                // Finally, the source value is defined, and not an array, so we don't care about the update value
+                continue;
+            }
+        }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" &&
+                typeof value === "object" &&
+                !Array.isArray(source[key]) &&
+                !Array.isArray(value)) {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    /**
+     * Validates the transformed data item using existing validation logic.
+     *
+     * @param transformedData - The transformed data to validate
+     * @param attributeMappings - The attribute mappings containing validation rules
+     * @param context - The context for validation
+     * @returns True if valid, false otherwise
+     */
+    validateTransformedData(transformedData, attributeMappings, context) {
+        try {
+            return this.importDataActions.validateItem(transformedData, attributeMappings, context);
+        }
+        catch (error) {
+            logger.error(`Validation error for transformed data: ${JSON.stringify(transformedData, null, 2)}`, error);
+            return false;
+        }
+    }
+}

package/dist/migrations/services/FileHandlerService.d.ts

@@ -0,0 +1,75 @@
+import type { AttributeMappings, AppwriteConfig } from "@njdamstra/appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+import { RateLimitManager } from "./RateLimitManager.js";
+/**
+ * Service responsible for file handling during import.
+ * Preserves all existing file handling capabilities including URL support.
+ * Extracted from DataLoader to provide focused, testable file operations.
+ */
+export declare class FileHandlerService {
+    private appwriteFolderPath;
+    private config;
+    private importDataActions;
+    private rateLimitManager;
+    constructor(appwriteFolderPath: string, config: AppwriteConfig, importDataActions: ImportDataActions, rateLimitManager?: RateLimitManager);
+    /**
+     * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+     * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+     * and update the field with the file's ID if necessary.
+     *
+     * Preserves existing file handling logic from DataLoader.
+     *
+     * @param attributeMappings - The attribute mappings from the import definition.
+     * @param context - The context object containing information about the database, collection, and document.
+     * @param item - The item being imported, used for resolving template paths in fileData mappings.
+     * @returns The attribute mappings updated with any necessary post-import actions.
+     */
+    getAttributeMappingsWithFileActions(attributeMappings: AttributeMappings, context: any, item: any): AttributeMappings;
+    /**
+     * Resolves local file path, searching in subdirectories if needed.
+     * Preserves existing file search logic from DataLoader.
+     *
+     * @param mappingFilePath - The relative file path from the mapping
+     * @returns Resolved absolute file path
+     */
+    private resolveLocalFilePath;
+    /**
+     * Executes file-related post-import actions with rate limiting.
+     * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
+     *
+     * @param context - The context containing document and collection information
+     * @param postImportActions - The post-import actions to execute
+     */
+    executeFileActions(context: any, postImportActions: any[]): Promise<void>;
+    /**
+     * Executes a single file action with proper error handling.
+     *
+     * @param action - The file action to execute
+     * @param context - The context for template resolution
+     */
+    private executeFileAction;
+    /**
+     * Validates that file paths exist before import begins.
+     * Provides early validation to catch file issues before processing starts.
+     *
+     * @param attributeMappings - The attribute mappings to validate
+     * @param context - The context for template resolution
+     * @param item - The item for template resolution
+     * @returns Array of validation errors (empty if all valid)
+     */
+    validateFilePaths(attributeMappings: AttributeMappings, context: any, item: any): string[];
+    /**
+     * Gets file statistics for import planning.
+     * Helps estimate import time and resource requirements.
+     *
+     * @param attributeMappings - The attribute mappings to analyze
+     * @param items - The items that will be imported
+     * @returns File statistics object
+     */
+    getFileStatistics(attributeMappings: AttributeMappings, items: any[]): {
+        totalFiles: number;
+        urlFiles: number;
+        localFiles: number;
+        estimatedSize: number;
+    };
+}