appwrite-utils-cli 0.10.86 → 1.0.2

package/dist/migrations/transfer.js

@@ -6,10 +6,12 @@ import { createOrUpdateAttribute, createUpdateCollectionAttributes, } from "../c
 import { parseAttribute } from "appwrite-utils";
 import chalk from "chalk";
 import { fetchAllCollections } from "../collections/methods.js";
+import { MessageFormatter } from "../shared/messageFormatter.js";
+import { ProgressManager } from "../shared/progressManager.js";
 import { createOrUpdateIndex, createOrUpdateIndexes, } from "../collections/indexes.js";
 import { getClient } from "../utils/getClientFromConfig.js";
 export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucketId) => {
-    console.log(`Transferring files from ${fromBucketId} to ${toBucketId}`);
+    MessageFormatter.info(`Transferring files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
     let lastFileId;
     let fromFiles = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [Query.limit(100)]));
     const allFromFiles = fromFiles.files;
@@ -21,7 +23,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
                 return await storage.getFileDownload(bucketId, fileId);
             }
             catch (error) {
-                console.error(`Error downloading file ${fileId}: ${error}`);
+                MessageFormatter.error(`Error downloading file ${fileId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
                 attempts--;
                 if (attempts === 0)
                     throw error;
@@ -32,11 +34,11 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
         for (const file of allFromFiles) {
             const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
             if (!fileData) {
-                console.error(`Error downloading file ${file.$id}`);
+                MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
                 continue;
             }
             const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
-            console.log(`Creating file: ${file.name}`);
+            MessageFormatter.progress(`Creating file: ${file.name}`, { prefix: "Transfer" });
             try {
                 await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
             }
@@ -65,7 +67,7 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
         for (const file of allFromFiles) {
             const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
             if (!fileData) {
-                console.error(`Error downloading file ${file.$id}`);
+                MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
                 continue;
             }
             const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
@@ -74,16 +76,16 @@ export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucke
             }
             catch (error) {
                 // File already exists, so we can skip it
-                console.log(chalk.yellow(`File ${file.$id} already exists, skipping...`));
+                MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
                 continue;
             }
             numberOfFiles++;
         }
     }
-    console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
+    MessageFormatter.success(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
 };
 export const transferStorageLocalToRemote = async (localStorage, endpoint, projectId, apiKey, fromBucketId, toBucketId) => {
-    console.log(`Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`);
+    MessageFormatter.info(`Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`, { prefix: "Transfer" });
     const client = getAppwriteClient(endpoint, projectId, apiKey);
     const remoteStorage = new Storage(client);
     let numberOfFiles = 0;
@@ -112,135 +114,15 @@ export const transferStorageLocalToRemote = async (localStorage, endpoint, proje
         }
         catch (error) {
             // File already exists, so we can skip it
-            console.log(chalk.yellow(`File ${file.$id} already exists, skipping...`));
+            MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
             continue;
         }
         numberOfFiles++;
     }
-    console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
-};
-/**
- * Transfers all documents from one collection to another in a different database
- * within the same Appwrite Project
- */
-export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
-    let totalDocumentsTransferred = 0;
-    let lastDocumentId;
-    let hasMoreDocuments = true;
-    while (hasMoreDocuments) {
-        const queryParams = [Query.limit(50)];
-        if (lastDocumentId) {
-            queryParams.push(Query.cursorAfter(lastDocumentId));
-        }
-        const fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, queryParams));
-        if (fromCollDocs.documents.length === 0) {
-            if (totalDocumentsTransferred === 0) {
-                console.log(`No documents found in collection ${fromCollId}`);
-            }
-            break;
-        }
-        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await db.listDocuments(toDbId, toCollId, [
-            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
-        ]));
-        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
-        const batchedPromises = docsToCreate.map((doc) => {
-            const toCreateObject = {
-                ...doc,
-            };
-            delete toCreateObject.$databaseId;
-            delete toCreateObject.$collectionId;
-            delete toCreateObject.$createdAt;
-            delete toCreateObject.$updatedAt;
-            delete toCreateObject.$id;
-            delete toCreateObject.$permissions;
-            return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-        });
-        await Promise.all(batchedPromises);
-        totalDocumentsTransferred += docsToCreate.length;
-        if (fromCollDocs.documents.length < 50) {
-            hasMoreDocuments = false;
-        }
-        else {
-            lastDocumentId =
-                fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
-        }
-    }
-    console.log(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`);
-};
-export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId, fromCollId, toCollId) => {
-    const client = new Client()
-        .setEndpoint(endpoint)
-        .setProject(projectId)
-        .setKey(apiKey);
-    let totalDocumentsTransferred = 0;
-    const remoteDb = new Databases(client);
-    let fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
-    if (fromCollDocs.documents.length === 0) {
-        console.log(`No documents found in collection ${fromCollId}`);
-        return;
-    }
-    else if (fromCollDocs.documents.length < 50) {
-        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await remoteDb.listDocuments(toDbId, toCollId, [
-            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
-        ]));
-        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
-        const batchedPromises = docsToCreate.map((doc) => {
-            const toCreateObject = {
-                ...doc,
-            };
-            delete toCreateObject.$databaseId;
-            delete toCreateObject.$collectionId;
-            delete toCreateObject.$createdAt;
-            delete toCreateObject.$updatedAt;
-            delete toCreateObject.$id;
-            delete toCreateObject.$permissions;
-            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-        });
-        await Promise.all(batchedPromises);
-        totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
-    else {
-        const allDocsToCreateCheck = await tryAwaitWithRetry(async () => await remoteDb.listDocuments(toDbId, toCollId, [
-            Query.equal("$id", fromCollDocs.documents.map((doc) => doc.$id)),
-        ]));
-        const docsToCreate = fromCollDocs.documents.filter((doc) => !allDocsToCreateCheck.documents.some((d) => d.$id === doc.$id));
-        const batchedPromises = docsToCreate.map((doc) => {
-            const toCreateObject = {
-                ...doc,
-            };
-            delete toCreateObject.$databaseId;
-            delete toCreateObject.$collectionId;
-            delete toCreateObject.$createdAt;
-            delete toCreateObject.$updatedAt;
-            delete toCreateObject.$id;
-            delete toCreateObject.$permissions;
-            return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-        });
-        await Promise.all(batchedPromises);
-        totalDocumentsTransferred += fromCollDocs.documents.length;
-        while (fromCollDocs.documents.length === 50) {
-            fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [
-                Query.limit(50),
-                Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
-            ]));
-            const batchedPromises = fromCollDocs.documents.map((doc) => {
-                const toCreateObject = {
-                    ...doc,
-                };
-                delete toCreateObject.$databaseId;
-                delete toCreateObject.$collectionId;
-                delete toCreateObject.$createdAt;
-                delete toCreateObject.$updatedAt;
-                delete toCreateObject.$id;
-                delete toCreateObject.$permissions;
-                return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
-            });
-            await Promise.all(batchedPromises);
-            totalDocumentsTransferred += fromCollDocs.documents.length;
-        }
-    }
-    console.log(`Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`);
+    MessageFormatter.success(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
 };
+// Document transfer functions moved to collections/methods.ts with enhanced UX
+// Remote document transfer functions moved to collections/methods.ts with enhanced UX
 /**
  * Transfers all collections and documents from one local database to another local database.
  *
@@ -307,6 +189,7 @@ export const transferDatabaseLocalToLocal = async (localDb, fromDbId, targetDbId
                 }
             }
             // Transfer documents
+            const { transferDocumentsBetweenDbsLocalToLocal } = await import("../collections/methods.js");
             await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, targetCollection.$id);
         }
         catch (error) {
@@ -371,6 +254,7 @@ export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId
                 }
             }
             // Transfer documents
+            const { transferDocumentsBetweenDbsLocalToRemote } = await import("../collections/methods.js");
             await transferDocumentsBetweenDbsLocalToRemote(localDb, endpoint, projectId, apiKey, fromDbId, toDbId, collection.$id, targetCollection.$id);
         }
         catch (error) {

package/dist/migrations/yaml/YamlImportConfigLoader.d.ts

@@ -0,0 +1,384 @@
+import { z } from "zod";
+import type { ImportDef } from "appwrite-utils";
+export declare const YamlImportConfigSchema: z.ZodObject<{
+    source: z.ZodObject<{
+        file: z.ZodString;
+        basePath: z.ZodOptional<z.ZodString>;
+        type: z.ZodDefault<z.ZodEnum<["json", "csv", "yaml"]>>;
+    }, "strip", z.ZodTypeAny, {
+        type: "json" | "csv" | "yaml";
+        file: string;
+        basePath?: string | undefined;
+    }, {
+        file: string;
+        type?: "json" | "csv" | "yaml" | undefined;
+        basePath?: string | undefined;
+    }>;
+    target: z.ZodObject<{
+        collection: z.ZodString;
+        type: z.ZodDefault<z.ZodEnum<["create", "update"]>>;
+        primaryKey: z.ZodDefault<z.ZodString>;
+        createUsers: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        type: "create" | "update";
+        collection: string;
+        primaryKey: string;
+        createUsers: boolean;
+    }, {
+        collection: string;
+        type?: "create" | "update" | undefined;
+        primaryKey?: string | undefined;
+        createUsers?: boolean | undefined;
+    }>;
+    mapping: z.ZodObject<{
+        attributes: z.ZodArray<z.ZodObject<{
+            oldKey: z.ZodOptional<z.ZodString>;
+            oldKeys: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            targetKey: z.ZodString;
+            valueToSet: z.ZodOptional<z.ZodAny>;
+            fileData: z.ZodOptional<z.ZodObject<{
+                path: z.ZodString;
+                name: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                path: string;
+                name: string;
+            }, {
+                path: string;
+                name: string;
+            }>>;
+            converters: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+            validation: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                rule: z.ZodString;
+                params: z.ZodArray<z.ZodString, "many">;
+            }, "strip", z.ZodTypeAny, {
+                params: string[];
+                rule: string;
+            }, {
+                params: string[];
+                rule: string;
+            }>, "many">>;
+            afterImport: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                action: z.ZodString;
+                params: z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>]>, "many">;
+            }, "strip", z.ZodTypeAny, {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }, {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }>, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            validation: {
+                params: string[];
+                rule: string;
+            }[];
+            targetKey: string;
+            converters: string[];
+            afterImport: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[];
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+        }, {
+            targetKey: string;
+            validation?: {
+                params: string[];
+                rule: string;
+            }[] | undefined;
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+            converters?: string[] | undefined;
+            afterImport?: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[] | undefined;
+        }>, "many">;
+        relationships: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            sourceField: z.ZodString;
+            targetField: z.ZodString;
+            targetCollection: z.ZodString;
+            fieldToSet: z.ZodOptional<z.ZodString>;
+            targetFieldToMatch: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }, {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }>, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        attributes: {
+            validation: {
+                params: string[];
+                rule: string;
+            }[];
+            targetKey: string;
+            converters: string[];
+            afterImport: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[];
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+        }[];
+        relationships: {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }[];
+    }, {
+        attributes: {
+            targetKey: string;
+            validation?: {
+                params: string[];
+                rule: string;
+            }[] | undefined;
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+            converters?: string[] | undefined;
+            afterImport?: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[] | undefined;
+        }[];
+        relationships?: {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }[] | undefined;
+    }>;
+    options: z.ZodDefault<z.ZodObject<{
+        batchSize: z.ZodDefault<z.ZodNumber>;
+        skipValidation: z.ZodDefault<z.ZodBoolean>;
+        dryRun: z.ZodDefault<z.ZodBoolean>;
+        continueOnError: z.ZodDefault<z.ZodBoolean>;
+        updateMapping: z.ZodOptional<z.ZodObject<{
+            originalIdField: z.ZodString;
+            targetField: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            targetField: string;
+            originalIdField: string;
+        }, {
+            targetField: string;
+            originalIdField: string;
+        }>>;
+    }, "strip", z.ZodTypeAny, {
+        batchSize: number;
+        skipValidation: boolean;
+        dryRun: boolean;
+        continueOnError: boolean;
+        updateMapping?: {
+            targetField: string;
+            originalIdField: string;
+        } | undefined;
+    }, {
+        batchSize?: number | undefined;
+        skipValidation?: boolean | undefined;
+        dryRun?: boolean | undefined;
+        continueOnError?: boolean | undefined;
+        updateMapping?: {
+            targetField: string;
+            originalIdField: string;
+        } | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    options: {
+        batchSize: number;
+        skipValidation: boolean;
+        dryRun: boolean;
+        continueOnError: boolean;
+        updateMapping?: {
+            targetField: string;
+            originalIdField: string;
+        } | undefined;
+    };
+    target: {
+        type: "create" | "update";
+        collection: string;
+        primaryKey: string;
+        createUsers: boolean;
+    };
+    source: {
+        type: "json" | "csv" | "yaml";
+        file: string;
+        basePath?: string | undefined;
+    };
+    mapping: {
+        attributes: {
+            validation: {
+                params: string[];
+                rule: string;
+            }[];
+            targetKey: string;
+            converters: string[];
+            afterImport: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[];
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+        }[];
+        relationships: {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }[];
+    };
+}, {
+    target: {
+        collection: string;
+        type?: "create" | "update" | undefined;
+        primaryKey?: string | undefined;
+        createUsers?: boolean | undefined;
+    };
+    source: {
+        file: string;
+        type?: "json" | "csv" | "yaml" | undefined;
+        basePath?: string | undefined;
+    };
+    mapping: {
+        attributes: {
+            targetKey: string;
+            validation?: {
+                params: string[];
+                rule: string;
+            }[] | undefined;
+            oldKey?: string | undefined;
+            oldKeys?: string[] | undefined;
+            valueToSet?: any;
+            fileData?: {
+                path: string;
+                name: string;
+            } | undefined;
+            converters?: string[] | undefined;
+            afterImport?: {
+                params: (string | Record<string, any>)[];
+                action: string;
+            }[] | undefined;
+        }[];
+        relationships?: {
+            targetField: string;
+            sourceField: string;
+            targetCollection: string;
+            fieldToSet?: string | undefined;
+            targetFieldToMatch?: string | undefined;
+        }[] | undefined;
+    };
+    options?: {
+        batchSize?: number | undefined;
+        skipValidation?: boolean | undefined;
+        dryRun?: boolean | undefined;
+        continueOnError?: boolean | undefined;
+        updateMapping?: {
+            targetField: string;
+            originalIdField: string;
+        } | undefined;
+    } | undefined;
+}>;
+export type YamlImportConfig = z.infer<typeof YamlImportConfigSchema>;
+/**
+ * Service for loading and converting YAML import configurations.
+ * Integrates with existing .appwrite YAML structure while providing
+ * enhanced import configuration capabilities.
+ */
+export declare class YamlImportConfigLoader {
+    private appwriteFolderPath;
+    constructor(appwriteFolderPath: string);
+    /**
+     * Loads a YAML import configuration file.
+     *
+     * @param configPath - Path to the YAML config file relative to .appwrite/import/
+     * @returns Parsed and validated YAML import configuration
+     */
+    loadImportConfig(configPath: string): Promise<YamlImportConfig>;
+    /**
+     * Loads all import configurations from the .appwrite/import directory.
+     *
+     * @returns Map of collection names to their import configurations
+     */
+    loadAllImportConfigs(): Promise<Map<string, YamlImportConfig[]>>;
+    /**
+     * Converts YAML import configuration to legacy ImportDef format.
+     * Maintains compatibility with existing import system.
+     *
+     * @param yamlConfig - YAML import configuration
+     * @returns Legacy ImportDef object
+     */
+    convertToImportDef(yamlConfig: YamlImportConfig): ImportDef;
+    /**
+     * Generates a template YAML import configuration.
+     * Useful for getting started with YAML-based imports.
+     *
+     * @param collectionName - Name of the collection
+     * @param sourceFile - Source data file name
+     * @returns YAML configuration template
+     */
+    generateTemplate(collectionName: string, sourceFile: string): string;
+    /**
+     * Creates the import directory structure if it doesn't exist.
+     * Sets up the recommended directory layout for YAML import configurations.
+     */
+    createImportStructure(): Promise<void>;
+    /**
+     * Validates import configuration against collection schema.
+     * Ensures that all target keys exist as attributes in the collection.
+     *
+     * @param yamlConfig - YAML import configuration
+     * @param collectionAttributes - Collection attribute definitions
+     * @returns Validation errors (empty if valid)
+     */
+    validateAgainstCollection(yamlConfig: YamlImportConfig, collectionAttributes: any[]): string[];
+    /**
+     * Gets statistics about import configurations.
+     *
+     * @param configs - Map of collection configurations
+     * @returns Statistics object
+     */
+    getStatistics(configs: Map<string, YamlImportConfig[]>): {
+        totalConfigurations: number;
+        collectionsWithConfigs: number;
+        totalAttributeMappings: number;
+        totalRelationshipMappings: number;
+        configsByType: {
+            [type: string]: number;
+        };
+    };
+}