appwrite-utils-cli 0.10.86 → 1.0.2

package/dist/migrations/importController.js

@@ -1,12 +1,14 @@
 import { AppwriteException, ID, Query, } from "node-appwrite";
 import { areCollectionNamesSame, tryAwaitWithRetry } from "../utils/index.js";
 import { resolveAndUpdateRelationships } from "./relationships.js";
-import { UsersController } from "./users.js";
-import { logger } from "./logging.js";
-import { updateOperation } from "./migrationHelper.js";
-import { BatchSchema, OperationCreateSchema, OperationSchema, } from "./backup.js";
+import { UsersController } from "../users/methods.js";
+import { logger } from "../shared/logging.js";
+import { updateOperation } from "../shared/migrationHelpers.js";
+import { BatchSchema, OperationCreateSchema, OperationSchema, } from "../storage/schemas.js";
 import { DataLoader } from "./dataLoader.js";
 import { transferDatabaseLocalToLocal, transferStorageLocalToLocal, } from "./transfer.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { ProgressManager } from "../shared/progressManager.js";
 export class ImportController {
     config;
     database;
@@ -43,12 +45,10 @@ export class ImportController {
         let dataLoader;
         let databaseRan;
         for (let db of databasesToProcess) {
-            if (db.name.toLowerCase().trim().replace(" ", "") === "migrations") {
+            if (!this.config.useMigrations && db.name.toLowerCase().trim().replace(" ", "") === "migrations") {
                 continue;
             }
-            console.log(`---------------------------------`);
-            console.log(`Starting import data for database: ${db.name}`);
-            console.log(`---------------------------------`);
+            MessageFormatter.banner(`Starting import data for database: ${db.name}`, "Database Import");
             if (!databaseRan) {
                 databaseRan = db;
                 dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
@@ -172,10 +172,13 @@ export class ImportController {
                     // Skip further processing if no import operation is found
                     continue;
                 }
-                const importOperation = await this.database.getDocument("migrations", "currentOperations", importOperationId);
-                await updateOperation(this.database, importOperation.$id, {
-                    status: "in_progress",
-                });
+                let importOperation = null;
+                if (this.config.useMigrations) {
+                    importOperation = await this.database.getDocument("migrations", "currentOperations", importOperationId);
+                    await updateOperation(this.database, importOperation.$id, {
+                        status: "in_progress",
+                    }, this.config.useMigrations);
+                }
                 const collectionData = dataLoader.importMap.get(dataLoader.getCollectionKey(collection.name));
                 console.log(`Processing collection: ${collection.name}...`);
                 if (!collectionData) {
@@ -212,14 +215,18 @@ export class ImportController {
                     // Wait for all promises in the current batch to resolve
                     await Promise.all(batchPromises);
                     console.log(`Completed batch ${i + 1} of ${dataSplit.length}`);
-                    await updateOperation(this.database, importOperation.$id, {
-                        progress: processedItems,
-                    });
+                    if (this.config.useMigrations && importOperation) {
+                        await updateOperation(this.database, importOperation.$id, {
+                            progress: processedItems,
+                        }, this.config.useMigrations);
+                    }
                 }
                 // After all batches are processed, update the operation status to completed
-                await updateOperation(this.database, importOperation.$id, {
-                    status: "completed",
-                });
+                if (this.config.useMigrations && importOperation) {
+                    await updateOperation(this.database, importOperation.$id, {
+                        status: "completed",
+                    }, this.config.useMigrations);
+                }
             }
         }
     }

package/dist/migrations/importDataActions.js

@@ -1,10 +1,10 @@
 import {} from "node-appwrite";
 import { validationRules, } from "appwrite-utils";
 import { converterFunctions } from "appwrite-utils";
-import { convertObjectBySchema } from "./converters.js";
+import { convertObjectBySchema } from "../utils/dataConverters.js";
 import {} from "appwrite-utils";
 import { afterImportActions } from "./afterImportActions.js";
-import { logger } from "./logging.js";
+import { logger } from "../shared/logging.js";
 import { tryAwaitWithRetry } from "../utils/helperFunctions.js";
 export class ImportDataActions {
     db;

package/dist/migrations/logging.d.ts

@@ -1,2 +1,10 @@
 import winston from "winston";
-export declare const logger: winston.Logger;
+export interface LoggingConfig {
+    enabled: boolean;
+    level: string;
+    logDirectory?: string;
+    console: boolean;
+}
+export declare const configureLogging: (config?: Partial<LoggingConfig>) => void;
+export declare let logger: winston.Logger;
+export declare const updateLogger: () => void;

package/dist/migrations/logging.js

@@ -1,27 +1,46 @@
 import winston from "winston";
 import fs from "fs";
 import path from "path";
-// Ensure the logs directory exists
-const logDir = path.join(process.cwd(), "zlogs");
-if (!fs.existsSync(logDir)) {
-    fs.mkdirSync(logDir);
-}
-export const logger = winston.createLogger({
-    level: "debug",
-    format: winston.format.json({ space: 2 }),
-    defaultMeta: { service: "appwrite-utils-cli" },
-    transports: [
-        new winston.transports.File({
+const DEFAULT_LOGGING_CONFIG = {
+    enabled: false,
+    level: "info",
+    console: false,
+};
+let loggingConfig = DEFAULT_LOGGING_CONFIG;
+export const configureLogging = (config = {}) => {
+    loggingConfig = { ...DEFAULT_LOGGING_CONFIG, ...config };
+};
+const createLogger = () => {
+    const transports = [];
+    // Add console transport if enabled
+    if (loggingConfig.console) {
+        transports.push(new winston.transports.Console({
+            format: winston.format.combine(winston.format.colorize(), winston.format.simple())
+        }));
+    }
+    // Add file transports if logging is enabled
+    if (loggingConfig.enabled) {
+        const logDir = loggingConfig.logDirectory || path.join(process.cwd(), "zlogs");
+        if (!fs.existsSync(logDir)) {
+            fs.mkdirSync(logDir, { recursive: true });
+        }
+        transports.push(new winston.transports.File({
             filename: path.join(logDir, "error.log"),
             level: "error",
-        }),
-        new winston.transports.File({
-            filename: path.join(logDir, "warn.log"),
-            level: "warn",
-        }),
-        new winston.transports.File({
-            filename: path.join(logDir, "info.log"),
-            level: "info",
-        }),
-    ],
-});
+        }), new winston.transports.File({
+            filename: path.join(logDir, "combined.log"),
+        }));
+    }
+    return winston.createLogger({
+        level: loggingConfig.level,
+        format: winston.format.combine(winston.format.timestamp(), winston.format.errors({ stack: true }), winston.format.json()),
+        defaultMeta: { service: "appwrite-utils-cli" },
+        transports,
+        silent: !loggingConfig.enabled && !loggingConfig.console,
+    });
+};
+export let logger = createLogger();
+// Recreate logger when config changes
+export const updateLogger = () => {
+    logger = createLogger();
+};

package/dist/migrations/migrationHelper.d.ts

@@ -142,11 +142,11 @@ export type ContextObject = z.infer<typeof ContextObject>;
 export declare const createOrFindAfterImportOperation: (database: Databases, collectionId: string, context: ContextObject) => Promise<void>;
 export declare const addBatch: (database: Databases, data: string) => Promise<string>;
 export declare const getAfterImportOperations: (database: Databases, collectionId: string) => Promise<{
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
+    error: string;
     $id: string;
     $createdAt: string;
     $updatedAt: string;
-    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
-    error: string;
     collectionId: string;
     operationType: string;
     progress: number;
@@ -155,11 +155,11 @@ export declare const getAfterImportOperations: (database: Databases, collectionI
     batches?: string[] | undefined;
 }[]>;
 export declare const findOrCreateOperation: (database: Databases, collectionId: string, operationType: string, additionalQueries?: string[]) => Promise<{
+    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
+    error: string;
     $id: string;
     $createdAt: string;
     $updatedAt: string;
-    status: "error" | "pending" | "ready" | "in_progress" | "completed" | "cancelled";
-    error: string;
     collectionId: string;
     operationType: string;
     progress: number;

package/dist/migrations/relationships.js

@@ -1,6 +1,6 @@
 import { Databases, Query } from "node-appwrite";
 import { fetchAllCollections } from "../collections/methods.js";
-import { logger } from "./logging.js";
+import { logger } from "../shared/logging.js";
 /**
  * Finds collections that have defined relationship attributes.
  */

package/dist/migrations/services/DataTransformationService.d.ts

@@ -0,0 +1,55 @@
+import type { AttributeMappings, ConfigDatabase, CollectionCreate } from "appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+/**
+ * Service responsible for data transformation during import.
+ * Extracted from DataLoader to provide focused, testable transformation logic.
+ */
+export declare class DataTransformationService {
+    private importDataActions;
+    constructor(importDataActions: ImportDataActions);
+    /**
+     * Transforms the given item based on the provided attribute mappings.
+     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+     *
+     * Preserves existing transformation logic from DataLoader.
+     *
+     * @param item - The item to be transformed.
+     * @param attributeMappings - The mappings that define how each attribute should be transformed.
+     * @returns The transformed item.
+     */
+    transformData(item: any, attributeMappings: AttributeMappings): any;
+    /**
+     * Creates a context object for data transformation.
+     * Preserves existing context creation logic from DataLoader.
+     *
+     * @param db - The database configuration
+     * @param collection - The collection configuration
+     * @param item - The raw item data
+     * @param docId - The document ID
+     * @returns Context object for transformation
+     */
+    createContext(db: ConfigDatabase, collection: CollectionCreate, item: any, docId: string): any;
+    /**
+     * Merges two objects by updating the source object with the target object's values.
+     * Preserves existing merge logic from DataLoader.
+     *
+     * It iterates through the target object's keys and updates the source object if:
+     * - The source object has the key.
+     * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
+     *
+     * @param source - The source object to be updated.
+     * @param update - The target object with values to update the source object.
+     * @returns The updated source object.
+     */
+    mergeObjects(source: any, update: any): any;
+    /**
+     * Validates the transformed data item using existing validation logic.
+     *
+     * @param transformedData - The transformed data to validate
+     * @param attributeMappings - The attribute mappings containing validation rules
+     * @param context - The context for validation
+     * @returns True if valid, false otherwise
+     */
+    validateTransformedData(transformedData: any, attributeMappings: AttributeMappings, context: any): boolean;
+}

package/dist/migrations/services/DataTransformationService.js

@@ -0,0 +1,158 @@
+import { convertObjectByAttributeMappings } from "../../utils/dataConverters.js";
+import { logger } from "../../shared/logging.js";
+/**
+ * Service responsible for data transformation during import.
+ * Extracted from DataLoader to provide focused, testable transformation logic.
+ */
+export class DataTransformationService {
+    importDataActions;
+    constructor(importDataActions) {
+        this.importDataActions = importDataActions;
+    }
+    /**
+     * Transforms the given item based on the provided attribute mappings.
+     * This method applies conversion rules to the item's attributes as defined in the attribute mappings.
+     *
+     * Preserves existing transformation logic from DataLoader.
+     *
+     * @param item - The item to be transformed.
+     * @param attributeMappings - The mappings that define how each attribute should be transformed.
+     * @returns The transformed item.
+     */
+    transformData(item, attributeMappings) {
+        try {
+            // Convert the item using the attribute mappings provided
+            const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
+            // Run additional converter functions on the converted item, if any
+            return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
+        }
+        catch (error) {
+            logger.error(`Error transforming data for item: ${JSON.stringify(item, null, 2)}`, error);
+            throw error;
+        }
+    }
+    /**
+     * Creates a context object for data transformation.
+     * Preserves existing context creation logic from DataLoader.
+     *
+     * @param db - The database configuration
+     * @param collection - The collection configuration
+     * @param item - The raw item data
+     * @param docId - The document ID
+     * @returns Context object for transformation
+     */
+    createContext(db, collection, item, docId) {
+        return {
+            ...item, // Spread the item data for easy access to its properties
+            dbId: db.$id,
+            dbName: db.name,
+            collId: collection.$id,
+            collName: collection.name,
+            docId: docId,
+            createdDoc: {}, // Initially empty, to be updated when the document is created
+        };
+    }
+    /**
+     * Merges two objects by updating the source object with the target object's values.
+     * Preserves existing merge logic from DataLoader.
+     *
+     * It iterates through the target object's keys and updates the source object if:
+     * - The source object has the key.
+     * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
+     *
+     * @param source - The source object to be updated.
+     * @param update - The target object with values to update the source object.
+     * @returns The updated source object.
+     */
+    mergeObjects(source, update) {
+        // Create a new object to hold the merged result
+        const result = { ...source };
+        // Loop through the keys of the object we care about
+        for (const [key, value] of Object.entries(source)) {
+            // Check if the key exists in the target object
+            if (!Object.hasOwn(update, key)) {
+                // If the key doesn't exist, we can just skip it
+                continue;
+            }
+            if (update[key] === value) {
+                continue;
+            }
+            // If the value ain't here, we can just do whatever man
+            if (value === undefined || value === null || value === "") {
+                // If the update key is defined
+                if (update[key] !== undefined &&
+                    update[key] !== null &&
+                    update[key] !== "") {
+                    // might as well use it eh?
+                    result[key] = update[key];
+                }
+                // ELSE if the value is an array, because it would then not be === to those things above
+            }
+            else if (Array.isArray(value)) {
+                // Get the update value
+                const updateValue = update[key];
+                // If the update value is an array, concatenate and remove duplicates
+                // and poopy data
+                if (Array.isArray(updateValue)) {
+                    result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+                else {
+                    // If the update value is not an array, just use it
+                    result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+            }
+            else if (typeof value === "object" && !Array.isArray(value)) {
+                // If the value is an object, we need to merge it
+                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
+                    result[key] = this.mergeObjects(value, update[key]);
+                }
+            }
+            else {
+                // Finally, the source value is defined, and not an array, so we don't care about the update value
+                continue;
+            }
+        }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" &&
+                typeof value === "object" &&
+                !Array.isArray(source[key]) &&
+                !Array.isArray(value)) {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    /**
+     * Validates the transformed data item using existing validation logic.
+     *
+     * @param transformedData - The transformed data to validate
+     * @param attributeMappings - The attribute mappings containing validation rules
+     * @param context - The context for validation
+     * @returns True if valid, false otherwise
+     */
+    validateTransformedData(transformedData, attributeMappings, context) {
+        try {
+            return this.importDataActions.validateItem(transformedData, attributeMappings, context);
+        }
+        catch (error) {
+            logger.error(`Validation error for transformed data: ${JSON.stringify(transformedData, null, 2)}`, error);
+            return false;
+        }
+    }
+}

package/dist/migrations/services/FileHandlerService.d.ts

@@ -0,0 +1,75 @@
+import type { AttributeMappings, AppwriteConfig } from "appwrite-utils";
+import type { ImportDataActions } from "../importDataActions.js";
+import { RateLimitManager } from "./RateLimitManager.js";
+/**
+ * Service responsible for file handling during import.
+ * Preserves all existing file handling capabilities including URL support.
+ * Extracted from DataLoader to provide focused, testable file operations.
+ */
+export declare class FileHandlerService {
+    private appwriteFolderPath;
+    private config;
+    private importDataActions;
+    private rateLimitManager;
+    constructor(appwriteFolderPath: string, config: AppwriteConfig, importDataActions: ImportDataActions, rateLimitManager?: RateLimitManager);
+    /**
+     * Generates attribute mappings with post-import actions based on the provided attribute mappings.
+     * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
+     * and update the field with the file's ID if necessary.
+     *
+     * Preserves existing file handling logic from DataLoader.
+     *
+     * @param attributeMappings - The attribute mappings from the import definition.
+     * @param context - The context object containing information about the database, collection, and document.
+     * @param item - The item being imported, used for resolving template paths in fileData mappings.
+     * @returns The attribute mappings updated with any necessary post-import actions.
+     */
+    getAttributeMappingsWithFileActions(attributeMappings: AttributeMappings, context: any, item: any): AttributeMappings;
+    /**
+     * Resolves local file path, searching in subdirectories if needed.
+     * Preserves existing file search logic from DataLoader.
+     *
+     * @param mappingFilePath - The relative file path from the mapping
+     * @returns Resolved absolute file path
+     */
+    private resolveLocalFilePath;
+    /**
+     * Executes file-related post-import actions with rate limiting.
+     * Wraps the existing createFileAndUpdateField action with proper error handling and rate limiting.
+     *
+     * @param context - The context containing document and collection information
+     * @param postImportActions - The post-import actions to execute
+     */
+    executeFileActions(context: any, postImportActions: any[]): Promise<void>;
+    /**
+     * Executes a single file action with proper error handling.
+     *
+     * @param action - The file action to execute
+     * @param context - The context for template resolution
+     */
+    private executeFileAction;
+    /**
+     * Validates that file paths exist before import begins.
+     * Provides early validation to catch file issues before processing starts.
+     *
+     * @param attributeMappings - The attribute mappings to validate
+     * @param context - The context for template resolution
+     * @param item - The item for template resolution
+     * @returns Array of validation errors (empty if all valid)
+     */
+    validateFilePaths(attributeMappings: AttributeMappings, context: any, item: any): string[];
+    /**
+     * Gets file statistics for import planning.
+     * Helps estimate import time and resource requirements.
+     *
+     * @param attributeMappings - The attribute mappings to analyze
+     * @param items - The items that will be imported
+     * @returns File statistics object
+     */
+    getFileStatistics(attributeMappings: AttributeMappings, items: any[]): {
+        totalFiles: number;
+        urlFiles: number;
+        localFiles: number;
+        estimatedSize: number;
+    };
+}