appwrite-utils-cli 0.10.85 → 1.0.1

package/dist/migrations/yaml/YamlImportConfigLoader.js

@@ -0,0 +1,375 @@
+import { z } from "zod";
+import path from "path";
+import fs from "fs";
+import yaml from "js-yaml";
+import { logger } from "../../shared/logging.js";
+// YAML Import Configuration Schema
+export const YamlImportConfigSchema = z.object({
+    source: z.object({
+        file: z.string().describe("Path to the data file relative to .appwrite directory"),
+        basePath: z.string().optional().describe("JSON path to the data array (e.g., 'RECORDS')"),
+        type: z.enum(["json", "csv", "yaml"]).default("json").describe("Source file type"),
+    }),
+    target: z.object({
+        collection: z.string().describe("Name of the target collection"),
+        type: z.enum(["create", "update"]).default("create").describe("Import operation type"),
+        primaryKey: z.string().default("id").describe("Primary key field name in source data"),
+        createUsers: z.boolean().default(false).describe("Whether to create user accounts"),
+    }),
+    mapping: z.object({
+        attributes: z.array(z.object({
+            // Source mapping
+            oldKey: z.string().optional().describe("Source field name"),
+            oldKeys: z.array(z.string()).optional().describe("Multiple source field names"),
+            // Target mapping
+            targetKey: z.string().describe("Target field name in collection"),
+            valueToSet: z.any().optional().describe("Static value to set"),
+            // File handling
+            fileData: z.object({
+                path: z.string().describe("File path template (supports {field} placeholders)"),
+                name: z.string().describe("File name template (supports {field} placeholders)"),
+            }).optional(),
+            // Data transformation
+            converters: z.array(z.string()).default([]).describe("Converter function names"),
+            // Validation
+            validation: z.array(z.object({
+                rule: z.string().describe("Validation rule name"),
+                params: z.array(z.string()).describe("Validation parameters with {field} placeholders"),
+            })).default([]).describe("Validation rules"),
+            // Post-import actions
+            afterImport: z.array(z.object({
+                action: z.string().describe("Action name"),
+                params: z.array(z.union([z.string(), z.record(z.string(), z.any())])).describe("Action parameters"),
+            })).default([]).describe("Actions to execute after import"),
+        })).describe("Field mapping configuration"),
+        relationships: z.array(z.object({
+            sourceField: z.string().describe("Source field containing old ID"),
+            targetField: z.string().describe("Target field to set new ID"),
+            targetCollection: z.string().describe("Collection to find new ID in"),
+            fieldToSet: z.string().optional().describe("Field to set (defaults to sourceField)"),
+            targetFieldToMatch: z.string().optional().describe("Field to match in target collection"),
+        })).default([]).describe("Relationship mappings"),
+    }),
+    options: z.object({
+        batchSize: z.number().min(1).max(1000).default(50).describe("Batch size for processing"),
+        skipValidation: z.boolean().default(false).describe("Skip data validation"),
+        dryRun: z.boolean().default(false).describe("Perform dry run without actual import"),
+        continueOnError: z.boolean().default(true).describe("Continue processing if individual items fail"),
+        updateMapping: z.object({
+            originalIdField: z.string().describe("Field in source data for matching"),
+            targetField: z.string().describe("Field in collection to match against"),
+        }).optional().describe("Configuration for update operations"),
+    }).default({}),
+});
+/**
+ * Service for loading and converting YAML import configurations.
+ * Integrates with existing .appwrite YAML structure while providing
+ * enhanced import configuration capabilities.
+ */
+export class YamlImportConfigLoader {
+    appwriteFolderPath;
+    constructor(appwriteFolderPath) {
+        this.appwriteFolderPath = appwriteFolderPath;
+    }
+    /**
+     * Loads a YAML import configuration file.
+     *
+     * @param configPath - Path to the YAML config file relative to .appwrite/import/
+     * @returns Parsed and validated YAML import configuration
+     */
+    async loadImportConfig(configPath) {
+        const fullPath = path.resolve(this.appwriteFolderPath, "import", configPath);
+        if (!fs.existsSync(fullPath)) {
+            throw new Error(`Import configuration file not found: ${fullPath}`);
+        }
+        try {
+            const yamlContent = fs.readFileSync(fullPath, "utf8");
+            const rawConfig = yaml.load(yamlContent);
+            // Validate against schema
+            const validatedConfig = YamlImportConfigSchema.parse(rawConfig);
+            logger.info(`Loaded import configuration: ${configPath}`);
+            return validatedConfig;
+        }
+        catch (error) {
+            if (error instanceof z.ZodError) {
+                const errorMessages = error.errors.map(err => `${err.path.join('.')}: ${err.message}`);
+                throw new Error(`Invalid import configuration in ${configPath}:\n${errorMessages.join('\n')}`);
+            }
+            throw new Error(`Failed to load import configuration ${configPath}: ${error}`);
+        }
+    }
+    /**
+     * Loads all import configurations from the .appwrite/import directory.
+     *
+     * @returns Map of collection names to their import configurations
+     */
+    async loadAllImportConfigs() {
+        const importDir = path.join(this.appwriteFolderPath, "import");
+        const configs = new Map();
+        if (!fs.existsSync(importDir)) {
+            logger.info("No import directory found, skipping YAML import configurations");
+            return configs;
+        }
+        try {
+            const files = fs.readdirSync(importDir, { withFileTypes: true });
+            for (const file of files) {
+                if (file.isFile() && file.name.endsWith('.yaml')) {
+                    try {
+                        const config = await this.loadImportConfig(file.name);
+                        const collectionName = config.target.collection;
+                        if (!configs.has(collectionName)) {
+                            configs.set(collectionName, []);
+                        }
+                        configs.get(collectionName).push(config);
+                    }
+                    catch (error) {
+                        logger.error(`Failed to load import config ${file.name}:`, error);
+                    }
+                }
+            }
+            logger.info(`Loaded import configurations for ${configs.size} collections`);
+            return configs;
+        }
+        catch (error) {
+            logger.error("Failed to scan import directory:", error);
+            return configs;
+        }
+    }
+    /**
+     * Converts YAML import configuration to legacy ImportDef format.
+     * Maintains compatibility with existing import system.
+     *
+     * @param yamlConfig - YAML import configuration
+     * @returns Legacy ImportDef object
+     */
+    convertToImportDef(yamlConfig) {
+        const attributeMappings = yamlConfig.mapping.attributes.map(attr => ({
+            oldKey: attr.oldKey,
+            oldKeys: attr.oldKeys,
+            targetKey: attr.targetKey,
+            valueToSet: attr.valueToSet,
+            fileData: attr.fileData,
+            converters: attr.converters,
+            validationActions: attr.validation.map(v => ({
+                action: v.rule,
+                params: v.params,
+            })),
+            postImportActions: attr.afterImport.map(a => ({
+                action: a.action,
+                params: a.params,
+            })),
+        }));
+        const idMappings = yamlConfig.mapping.relationships.map(rel => ({
+            sourceField: rel.sourceField,
+            targetField: rel.targetField,
+            targetCollection: rel.targetCollection,
+            fieldToSet: rel.fieldToSet,
+            targetFieldToMatch: rel.targetFieldToMatch,
+        }));
+        return {
+            type: yamlConfig.target.type,
+            filePath: yamlConfig.source.file,
+            basePath: yamlConfig.source.basePath,
+            primaryKeyField: yamlConfig.target.primaryKey,
+            createUsers: yamlConfig.target.createUsers,
+            attributeMappings,
+            idMappings: idMappings.length > 0 ? idMappings : undefined,
+            updateMapping: yamlConfig.options.updateMapping,
+        };
+    }
+    /**
+     * Generates a template YAML import configuration.
+     * Useful for getting started with YAML-based imports.
+     *
+     * @param collectionName - Name of the collection
+     * @param sourceFile - Source data file name
+     * @returns YAML configuration template
+     */
+    generateTemplate(collectionName, sourceFile) {
+        const template = {
+            source: {
+                file: `importData/${sourceFile}`,
+                basePath: "RECORDS",
+                type: "json"
+            },
+            target: {
+                collection: collectionName,
+                type: "create",
+                primaryKey: "id",
+                createUsers: false
+            },
+            mapping: {
+                attributes: [
+                    {
+                        oldKey: "id",
+                        targetKey: "id",
+                        converters: ["anyToString"]
+                    },
+                    {
+                        oldKey: "name",
+                        targetKey: "name",
+                        converters: ["anyToString"],
+                        validation: [
+                            {
+                                rule: "required",
+                                params: ["{name}"]
+                            }
+                        ]
+                    },
+                    {
+                        oldKey: "avatar_url",
+                        targetKey: "avatar",
+                        fileData: {
+                            path: "{avatar_url}",
+                            name: "{name}_avatar"
+                        },
+                        afterImport: [
+                            {
+                                action: "createFileAndUpdateField",
+                                params: ["{dbId}", "{collId}", "{docId}", "avatar", "{bucketId}", "{filePath}", "{fileName}"]
+                            }
+                        ]
+                    }
+                ],
+                relationships: [
+                    {
+                        sourceField: "user_id",
+                        targetField: "userId",
+                        targetCollection: "Users"
+                    }
+                ]
+            },
+            options: {
+                batchSize: 50,
+                skipValidation: false,
+                dryRun: false,
+                continueOnError: true
+            }
+        };
+        return yaml.dump(template, {
+            indent: 2,
+            lineWidth: 120,
+            sortKeys: false,
+        });
+    }
+    /**
+     * Creates the import directory structure if it doesn't exist.
+     * Sets up the recommended directory layout for YAML import configurations.
+     */
+    async createImportStructure() {
+        const importDir = path.join(this.appwriteFolderPath, "import");
+        const collectionsDir = path.join(importDir, "collections");
+        const templatesDir = path.join(importDir, "templates");
+        // Create directories
+        for (const dir of [importDir, collectionsDir, templatesDir]) {
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+                logger.info(`Created directory: ${dir}`);
+            }
+        }
+        // Create README file
+        const readmePath = path.join(importDir, "README.md");
+        if (!fs.existsSync(readmePath)) {
+            const readmeContent = `# Import Configurations
+
+This directory contains YAML-based import configurations for the Appwrite Utils CLI.
+
+## Structure
+
+- \`collections/\` - Collection-specific import configurations
+- \`templates/\` - Template configurations for reference
+- \`*.yaml\` - Individual import configuration files
+
+## Configuration Format
+
+Each YAML file defines:
+- **source**: Data source configuration (file, type, basePath)
+- **target**: Target collection and operation type
+- **mapping**: Field mappings, transformations, and relationships
+- **options**: Import options (batch size, validation, etc.)
+
+## Example
+
+\`\`\`yaml
+source:
+  file: "importData/users.json"
+  basePath: "RECORDS"
+  type: "json"
+
+target:
+  collection: "Users"
+  type: "create"
+  primaryKey: "id"
+  createUsers: true
+
+mapping:
+  attributes:
+    - oldKey: "user_id"
+      targetKey: "userId"
+      converters: ["anyToString"]
+    - oldKey: "profile_image"
+      targetKey: "avatar"
+      fileData:
+        path: "assets/profiles/{user_id}.jpg"
+        name: "{firstName}_{lastName}_avatar"
+
+options:
+  batchSize: 50
+  continueOnError: true
+\`\`\`
+
+## Usage
+
+The CLI will automatically detect and load YAML import configurations during the import process.
+`;
+            fs.writeFileSync(readmePath, readmeContent);
+            logger.info(`Created README: ${readmePath}`);
+        }
+    }
+    /**
+     * Validates import configuration against collection schema.
+     * Ensures that all target keys exist as attributes in the collection.
+     *
+     * @param yamlConfig - YAML import configuration
+     * @param collectionAttributes - Collection attribute definitions
+     * @returns Validation errors (empty if valid)
+     */
+    validateAgainstCollection(yamlConfig, collectionAttributes) {
+        const errors = [];
+        const attributeKeys = new Set(collectionAttributes.map(attr => attr.key));
+        for (const mapping of yamlConfig.mapping.attributes) {
+            if (!attributeKeys.has(mapping.targetKey)) {
+                errors.push(`Target key '${mapping.targetKey}' not found in collection attributes`);
+            }
+        }
+        return errors;
+    }
+    /**
+     * Gets statistics about import configurations.
+     *
+     * @param configs - Map of collection configurations
+     * @returns Statistics object
+     */
+    getStatistics(configs) {
+        let totalConfigurations = 0;
+        let totalAttributeMappings = 0;
+        let totalRelationshipMappings = 0;
+        const configsByType = {};
+        for (const [collectionName, collectionConfigs] of configs.entries()) {
+            totalConfigurations += collectionConfigs.length;
+            for (const config of collectionConfigs) {
+                totalAttributeMappings += config.mapping.attributes.length;
+                totalRelationshipMappings += config.mapping.relationships.length;
+                const type = config.target.type;
+                configsByType[type] = (configsByType[type] || 0) + 1;
+            }
+        }
+        return {
+            totalConfigurations,
+            collectionsWithConfigs: configs.size,
+            totalAttributeMappings,
+            totalRelationshipMappings,
+            configsByType,
+        };
+    }
+}

package/dist/migrations/yaml/YamlImportIntegration.d.ts

@@ -0,0 +1,87 @@
+import type { CollectionCreate, ImportDef } from "appwrite-utils";
+/**
+ * Integration service that bridges YAML import configurations with the existing import system.
+ * Provides seamless integration between new YAML configs and legacy TypeScript collection definitions.
+ */
+export declare class YamlImportIntegration {
+    private configLoader;
+    private appwriteFolderPath;
+    constructor(appwriteFolderPath: string);
+    /**
+     * Initializes the YAML import system.
+     * Creates necessary directories, schemas, and example files.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Merges YAML import configurations with existing collection definitions.
+     * Allows collections to have both TypeScript and YAML import definitions.
+     *
+     * @param collections - Existing collection configurations
+     * @returns Collections with merged import definitions
+     */
+    mergeWithCollections(collections: CollectionCreate[]): Promise<CollectionCreate[]>;
+    /**
+     * Validates YAML import configurations against existing collection schemas.
+     * Ensures that all target fields exist in the collection definitions.
+     *
+     * @param collections - Collection definitions to validate against
+     * @returns Validation results with errors and warnings
+     */
+    validateConfigurations(collections: CollectionCreate[]): Promise<{
+        isValid: boolean;
+        errors: string[];
+        warnings: string[];
+    }>;
+    /**
+     * Validates that source files exist for import configurations.
+     */
+    private validateSourceFiles;
+    /**
+     * Validates that converter functions are available.
+     */
+    private validateConverters;
+    /**
+     * Generates a YAML import configuration from an existing ImportDef.
+     * Useful for migrating TypeScript configurations to YAML.
+     *
+     * @param importDef - Existing ImportDef to convert
+     * @param collectionName - Name of the collection
+     * @returns YAML configuration string
+     */
+    convertImportDefToYaml(importDef: ImportDef, collectionName: string): string;
+    /**
+     * Exports existing TypeScript import configurations to YAML files.
+     * Helps migrate from TypeScript to YAML-based configurations.
+     *
+     * @param collections - Collections with existing import definitions
+     */
+    exportToYaml(collections: CollectionCreate[]): Promise<void>;
+    /**
+     * Gets statistics about YAML import configurations.
+     */
+    getStatistics(): Promise<{
+        hasYamlConfigs: boolean;
+        totalConfigurations: number;
+        collectionsWithConfigs: number;
+        configurationsByType: {
+            [type: string]: number;
+        };
+        totalAttributeMappings: number;
+        totalRelationshipMappings: number;
+    }>;
+    /**
+     * Creates a new YAML import configuration from a template.
+     *
+     * @param collectionName - Name of the collection
+     * @param sourceFile - Source data file name
+     * @param outputPath - Output file path (optional)
+     */
+    createFromTemplate(collectionName: string, sourceFile: string, outputPath?: string): Promise<string>;
+    /**
+     * Checks if YAML import system is properly set up.
+     */
+    isSetupComplete(): Promise<{
+        isComplete: boolean;
+        missingComponents: string[];
+    }>;
+}