appwrite-utils-cli 1.9.6 → 1.11.0

package/dist/migrations/relationships.js

@@ -1,204 +0,0 @@
-import { Databases, Query } from "node-appwrite";
-import { fetchAllCollections } from "../collections/methods.js";
-import { logger } from "../shared/logging.js";
-import { MessageFormatter } from "../shared/messageFormatter.js";
-/**
- * Finds collections that have defined relationship attributes.
- */
-export const findCollectionsWithRelationships = (config) => {
-    const toReturn = new Map();
-    if (!config.collections) {
-        return toReturn;
-    }
-    for (const collection of config.collections) {
-        if (collection.attributes) {
-            for (const attribute of collection.attributes) {
-                if (attribute.type === "relationship" &&
-                    attribute.twoWay &&
-                    attribute.side === "parent") {
-                    toReturn.set(collection.name, toReturn.get(collection.name) || []);
-                    toReturn
-                        .get(collection.name)
-                        ?.push(attribute);
-                }
-            }
-        }
-    }
-    return toReturn;
-};
-export async function resolveAndUpdateRelationships(dbId, database, config) {
-    const collections = await fetchAllCollections(dbId, database);
-    const collectionsWithRelationships = findCollectionsWithRelationships(config);
-    // Process each collection sequentially
-    for (const collection of collections) {
-        MessageFormatter.processing(`Processing collection: ${collection.name} (${collection.$id})`, { prefix: "Migration" });
-        const relAttributeMap = collectionsWithRelationships.get(collection.name); // Get the relationship attributes for the collections
-        if (!relAttributeMap) {
-            MessageFormatter.info(`No mapping found for collection: ${collection.name}, skipping...`, { prefix: "Migration" });
-            continue;
-        }
-        await processCollection(dbId, database, collection, relAttributeMap);
-    }
-    MessageFormatter.success(`Completed relationship resolution and update for database ID: ${dbId}`, { prefix: "Migration" });
-}
-async function processCollection(dbId, database, collection, relAttributeMap) {
-    let after; // For pagination
-    let hasMore = true;
-    while (hasMore) {
-        const response = await database.listDocuments(dbId, collection.$id, [
-            Query.limit(100), // Fetch documents in batches of 100
-            ...(after ? [Query.cursorAfter(after)] : []),
-        ]);
-        const documents = response.documents;
-        MessageFormatter.info(`Fetched ${documents.length} documents from collection: ${collection.name}`, { prefix: "Migration" });
-        if (documents.length > 0) {
-            const updates = await prepareDocumentUpdates(database, dbId, collection.name, documents, relAttributeMap);
-            // Execute updates for the current batch
-            await executeUpdatesInBatches(dbId, database, updates);
-        }
-        if (documents.length === 100) {
-            after = documents[documents.length - 1].$id; // Prepare for the next page
-        }
-        else {
-            hasMore = false; // No more documents to fetch
-        }
-    }
-}
-async function findDocumentsByOriginalId(database, dbId, targetCollection, targetKey, originalId) {
-    const relatedCollectionId = targetCollection.$id;
-    const collection = await database.listCollections(dbId, [
-        Query.equal("$id", relatedCollectionId),
-    ]);
-    if (collection.total === 0) {
-        MessageFormatter.warning(`Collection ${relatedCollectionId} doesn't exist, skipping...`, { prefix: "Migration" });
-        return undefined;
-    }
-    const targetAttr = collection.collections[0].attributes.find(
-    // @ts-ignore
-    (attr) => attr.key === targetKey);
-    if (!targetAttr) {
-        MessageFormatter.warning(`Attribute ${targetKey} not found in collection ${relatedCollectionId}, skipping...`, { prefix: "Migration" });
-        return undefined;
-    }
-    let queries = [];
-    if (targetAttr.array) {
-        // @ts-ignore
-        queries.push(Query.contains(targetKey, originalId));
-    }
-    else {
-        queries.push(Query.equal(targetKey, originalId));
-    }
-    const response = await database.listDocuments(dbId, relatedCollectionId, [
-        ...queries,
-        Query.limit(500), // Adjust the limit based on your needs or implement pagination
-    ]);
-    if (response.documents.length < 0) {
-        return undefined;
-    }
-    else if (response.documents.length > 0) {
-        return response.documents;
-    }
-    else {
-        return undefined;
-    }
-}
-async function prepareDocumentUpdates(database, dbId, collectionName, documents, relationships) {
-    MessageFormatter.processing(`Preparing updates for collection: ${collectionName}`, { prefix: "Migration" });
-    const updates = [];
-    const thisCollection = (await database.listCollections(dbId, [Query.equal("name", collectionName)])).collections[0];
-    const thisCollectionId = thisCollection?.$id;
-    if (!thisCollectionId) {
-        MessageFormatter.warning(`No collection found with name: ${collectionName}`, { prefix: "Migration" });
-        return [];
-    }
-    for (const doc of documents) {
-        let updatePayload = {};
-        for (const rel of relationships) {
-            // Skip if not dealing with the parent side of a two-way relationship
-            if (rel.twoWay && rel.side !== "parent") {
-                MessageFormatter.info("Skipping non-parent side of two-way relationship...", { prefix: "Migration" });
-                continue;
-            }
-            const isSingleReference = rel.relationType === "oneToOne" || rel.relationType === "manyToOne";
-            const originalIdField = rel.importMapping?.originalIdField;
-            const targetField = rel.importMapping?.targetField || originalIdField; // Use originalIdField if targetField is not specified
-            if (!originalIdField) {
-                MessageFormatter.warning("Missing originalIdField in importMapping, skipping...", { prefix: "Migration" });
-                continue;
-            }
-            const originalId = doc[originalIdField];
-            if (!originalId) {
-                continue;
-            }
-            const relatedCollection = (await database.listCollections(dbId, [
-                Query.equal("name", rel.relatedCollection),
-            ])).collections[0];
-            if (!relatedCollection) {
-                MessageFormatter.warning(`Related collection ${rel.relatedCollection} not found, skipping...`, { prefix: "Migration" });
-                continue;
-            }
-            const foundDocuments = await findDocumentsByOriginalId(database, dbId, relatedCollection, targetField, String(originalId));
-            if (foundDocuments && foundDocuments.length > 0) {
-                const relationshipKey = rel.key;
-                const existingRefs = doc[relationshipKey] || [];
-                let existingRefIds = [];
-                if (Array.isArray(existingRefs)) {
-                    // @ts-ignore
-                    existingRefIds = existingRefs.map((ref) => ref.$id);
-                }
-                else if (existingRefs) {
-                    // @ts-ignore
-                    existingRefIds = [existingRefs.$id];
-                }
-                const newRefs = foundDocuments.map((fd) => fd.$id);
-                const allRefs = [...new Set([...existingRefIds, ...newRefs])]; // Combine and remove duplicates
-                // Update logic based on the relationship cardinality
-                updatePayload[relationshipKey] = isSingleReference
-                    ? newRefs[0] || existingRefIds[0]
-                    : allRefs;
-                MessageFormatter.info(`Updating ${relationshipKey} with ${allRefs.length} refs`, { prefix: "Migration" });
-            }
-        }
-        if (Object.keys(updatePayload).length > 0) {
-            updates.push({
-                collectionId: thisCollectionId,
-                documentId: doc.$id,
-                updatePayload: updatePayload,
-            });
-        }
-    }
-    return updates;
-}
-async function processInBatches(items, batchSize, processFunction) {
-    const maxParallelBatches = 25; // Adjust this value to control the number of parallel batches
-    let currentIndex = 0;
-    let activeBatchPromises = [];
-    while (currentIndex < items.length) {
-        // While there's still data to process and we haven't reached our parallel limit
-        while (currentIndex < items.length &&
-            activeBatchPromises.length < maxParallelBatches) {
-            const batch = items.slice(currentIndex, currentIndex + batchSize);
-            currentIndex += batchSize;
-            // Add new batch processing promise to the array
-            activeBatchPromises.push(processFunction(batch));
-        }
-        // Wait for one of the batch processes to complete
-        await Promise.race(activeBatchPromises).then(() => {
-            // Remove the resolved promise from the activeBatchPromises array
-            activeBatchPromises = activeBatchPromises.filter((p) => p !== Promise.race(activeBatchPromises));
-        });
-    }
-    // After processing all batches, ensure all active promises are resolved
-    await Promise.all(activeBatchPromises);
-}
-async function executeUpdatesInBatches(dbId, database, updates) {
-    const batchSize = 25; // Adjust based on your rate limit and performance testing
-    for (let i = 0; i < updates.length; i += batchSize) {
-        const batch = updates.slice(i, i + batchSize);
-        await Promise.all(batch.map((update) => database
-            .updateDocument(dbId, update.collectionId, update.documentId, update.updatePayload)
-            .catch((error) => {
-            logger.error(`Error updating doc ${update.documentId} in ${dbId}, update payload: ${JSON.stringify(update.updatePayload, undefined, 4)}, error: ${error}`);
-        })));
-    }
-}

package/dist/migrations/services/DataTransformationService.d.ts

@@ -1,55 +0,0 @@
-import type { AttributeMappings, ConfigDatabase, CollectionCreate } from "appwrite-utils";
-import type { ImportDataActions } from "../importDataActions.js";
-/**
- * Service responsible for data transformation during import.
- * Extracted from DataLoader to provide focused, testable transformation logic.
- */
-export declare class DataTransformationService {
-    private importDataActions;
-    constructor(importDataActions: ImportDataActions);
-    /**
-     * Transforms the given item based on the provided attribute mappings.
-     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
-     *
-     * Preserves existing transformation logic from DataLoader.
-     *
-     * @param item - The item to be transformed.
-     * @param attributeMappings - The mappings that define how each attribute should be transformed.
-     * @returns The transformed item.
-     */
-    transformData(item: any, attributeMappings: AttributeMappings): any;
-    /**
-     * Creates a context object for data transformation.
-     * Preserves existing context creation logic from DataLoader.
-     *
-     * @param db - The database configuration
-     * @param collection - The collection configuration
-     * @param item - The raw item data
-     * @param docId - The document ID
-     * @returns Context object for transformation
-     */
-    createContext(db: ConfigDatabase, collection: CollectionCreate, item: any, docId: string): any;
-    /**
-     * Merges two objects by updating the source object with the target object's values.
-     * Preserves existing merge logic from DataLoader.
-     *
-     * It iterates through the target object's keys and updates the source object if:
-     * - The source object has the key.
-     * - The target object's value for that key is not null, undefined, or an empty string.
-     * - If the target object has an array value, it concatenates the values and removes duplicates.
-     *
-     * @param source - The source object to be updated.
-     * @param update - The target object with values to update the source object.
-     * @returns The updated source object.
-     */
-    mergeObjects(source: any, update: any): any;
-    /**
-     * Validates the transformed data item using existing validation logic.
-     *
-     * @param transformedData - The transformed data to validate
-     * @param attributeMappings - The attribute mappings containing validation rules
-     * @param context - The context for validation
-     * @returns True if valid, false otherwise
-     */
-    validateTransformedData(transformedData: any, attributeMappings: AttributeMappings, context: any): boolean;
-}

package/dist/migrations/services/DataTransformationService.js

@@ -1,158 +0,0 @@
-import { convertObjectByAttributeMappings } from "../../utils/dataConverters.js";
-import { logger } from "../../shared/logging.js";
-/**
- * Service responsible for data transformation during import.
- * Extracted from DataLoader to provide focused, testable transformation logic.
- */
-export class DataTransformationService {
-    importDataActions;
-    constructor(importDataActions) {
-        this.importDataActions = importDataActions;
-    }
-    /**
-     * Transforms the given item based on the provided attribute mappings.
-     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
-     *
-     * Preserves existing transformation logic from DataLoader.
-     *
-     * @param item - The item to be transformed.
-     * @param attributeMappings - The mappings that define how each attribute should be transformed.
-     * @returns The transformed item.
-     */
-    transformData(item, attributeMappings) {
-        try {
-            // Convert the item using the attribute mappings provided
-            const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
-            // Run additional converter functions on the converted item, if any
-            return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
-        }
-        catch (error) {
-            logger.error(`Error transforming data for item: ${JSON.stringify(item, null, 2)}`, error);
-            throw error;
-        }
-    }
-    /**
-     * Creates a context object for data transformation.
-     * Preserves existing context creation logic from DataLoader.
-     *
-     * @param db - The database configuration
-     * @param collection - The collection configuration
-     * @param item - The raw item data
-     * @param docId - The document ID
-     * @returns Context object for transformation
-     */
-    createContext(db, collection, item, docId) {
-        return {
-            ...item, // Spread the item data for easy access to its properties
-            dbId: db.$id,
-            dbName: db.name,
-            collId: collection.$id,
-            collName: collection.name,
-            docId: docId,
-            createdDoc: {}, // Initially empty, to be updated when the document is created
-        };
-    }
-    /**
-     * Merges two objects by updating the source object with the target object's values.
-     * Preserves existing merge logic from DataLoader.
-     *
-     * It iterates through the target object's keys and updates the source object if:
-     * - The source object has the key.
-     * - The target object's value for that key is not null, undefined, or an empty string.
-     * - If the target object has an array value, it concatenates the values and removes duplicates.
-     *
-     * @param source - The source object to be updated.
-     * @param update - The target object with values to update the source object.
-     * @returns The updated source object.
-     */
-    mergeObjects(source, update) {
-        // Create a new object to hold the merged result
-        const result = { ...source };
-        // Loop through the keys of the object we care about
-        for (const [key, value] of Object.entries(source)) {
-            // Check if the key exists in the target object
-            if (!Object.hasOwn(update, key)) {
-                // If the key doesn't exist, we can just skip it
-                continue;
-            }
-            if (update[key] === value) {
-                continue;
-            }
-            // If the value ain't here, we can just do whatever man
-            if (value === undefined || value === null || value === "") {
-                // If the update key is defined
-                if (update[key] !== undefined &&
-                    update[key] !== null &&
-                    update[key] !== "") {
-                    // might as well use it eh?
-                    result[key] = update[key];
-                }
-                // ELSE if the value is an array, because it would then not be === to those things above
-            }
-            else if (Array.isArray(value)) {
-                // Get the update value
-                const updateValue = update[key];
-                // If the update value is an array, concatenate and remove duplicates
-                // and poopy data
-                if (Array.isArray(updateValue)) {
-                    result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
-                }
-                else {
-                    // If the update value is not an array, just use it
-                    result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
-                }
-            }
-            else if (typeof value === "object" && !Array.isArray(value)) {
-                // If the value is an object, we need to merge it
-                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
-                    result[key] = this.mergeObjects(value, update[key]);
-                }
-            }
-            else {
-                // Finally, the source value is defined, and not an array, so we don't care about the update value
-                continue;
-            }
-        }
-        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
-        for (const [key, value] of Object.entries(update)) {
-            if (value === undefined || value === null || value === "") {
-                continue;
-            }
-            else if (!Object.hasOwn(source, key)) {
-                result[key] = value;
-            }
-            else if (typeof source[key] === "object" &&
-                typeof value === "object" &&
-                !Array.isArray(source[key]) &&
-                !Array.isArray(value)) {
-                result[key] = this.mergeObjects(source[key], value);
-            }
-            else if (Array.isArray(source[key]) && Array.isArray(value)) {
-                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
-            }
-            else if (source[key] === undefined ||
-                source[key] === null ||
-                source[key] === "") {
-                result[key] = value;
-            }
-        }
-        return result;
-    }
-    /**
-     * Validates the transformed data item using existing validation logic.
-     *
-     * @param transformedData - The transformed data to validate
-     * @param attributeMappings - The attribute mappings containing validation rules
-     * @param context - The context for validation
-     * @returns True if valid, false otherwise
-     */
-    validateTransformedData(transformedData, attributeMappings, context) {
-        try {
-            return this.importDataActions.validateItem(transformedData, attributeMappings, context);
-        }
-        catch (error) {
-            logger.error(`Validation error for transformed data: ${JSON.stringify(transformedData, null, 2)}`, error);
-            return false;
-        }
-    }
-}

package/dist/migrations/services/FileHandlerService.d.ts

@@ -1,75 +0,0 @@
-import type { AttributeMappings, AppwriteConfig } from "appwrite-utils";
-import type { ImportDataActions } from "../importDataActions.js";
-import { RateLimitManager } from "./RateLimitManager.js";
-/**
- * Service responsible for file handling during import.
- * Preserves all existing file handling capabilities including URL support.
- * Extracted from DataLoader to provide focused, testable file operations.
- */
-export declare class FileHandlerService {
-    private appwriteFolderPath;
-    private config;
-    private importDataActions;
-    private rateLimitManager;
-    constructor(appwriteFolderPath: string, config: AppwriteConfig, importDataActions: ImportDataActions, rateLimitManager?: RateLimitManager);
-    /**
-     * Generates attribute mappings with post-import actions based on the provided attribute mappings.
-     * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
-     * and update the field with the file's ID if necessary.
-     *
-     * Preserves existing file handling logic from DataLoader.
-     *
-     * @param attributeMappings - The attribute mappings from the import definition.
-     * @param context - The context object containing information about the database, collection, and document.
-     * @param item - The item being imported, used for resolving template paths in fileData mappings.
-     * @returns The attribute mappings updated with any necessary post-import actions.
-     */
-    getAttributeMappingsWithFileActions(attributeMappings: AttributeMappings, context: any, item: any): AttributeMappings;
-    /**
-     * Resolves local file path, searching in subdirectories if needed.
-     * Preserves existing file search logic from DataLoader.
-     *
-     * @param mappingFilePath - The relative file path from the mapping
-     * @returns Resolved absolute file path
-     */
-    private resolveLocalFilePath;
-    /**
-     * Executes file-related post-import actions with rate limiting.
-     * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
-     *
-     * @param context - The context containing document and collection information
-     * @param postImportActions - The post-import actions to execute
-     */
-    executeFileActions(context: any, postImportActions: any[]): Promise<void>;
-    /**
-     * Executes a single file action with proper error handling.
-     *
-     * @param action - The file action to execute
-     * @param context - The context for template resolution
-     */
-    private executeFileAction;
-    /**
-     * Validates that file paths exist before import begins.
-     * Provides early validation to catch file issues before processing starts.
-     *
-     * @param attributeMappings - The attribute mappings to validate
-     * @param context - The context for template resolution
-     * @param item - The item for template resolution
-     * @returns Array of validation errors (empty if all valid)
-     */
-    validateFilePaths(attributeMappings: AttributeMappings, context: any, item: any): string[];
-    /**
-     * Gets file statistics for import planning.
-     * Helps estimate import time and resource requirements.
-     *
-     * @param attributeMappings - The attribute mappings to analyze
-     * @param items - The items that will be imported
-     * @returns File statistics object
-     */
-    getFileStatistics(attributeMappings: AttributeMappings, items: any[]): {
-        totalFiles: number;
-        urlFiles: number;
-        localFiles: number;
-        estimatedSize: number;
-    };
-}