@memberjunction/metadata-sync 2.47.0 → 2.49.0

package/dist/commands/pull/index.js

@@ -21,9 +21,10 @@ const path_1 = __importDefault(require("path"));
 const prompts_1 = require("@inquirer/prompts");
 const ora_classic_1 = __importDefault(require("ora-classic"));
 const config_1 = require("../../config");
-const sync_engine_1 = require("../../lib/sync-engine");
 const core_2 = require("@memberjunction/core");
 const provider_utils_1 = require("../../lib/provider-utils");
+const config_manager_1 = require("../../lib/config-manager");
+const singleton_manager_1 = require("../../lib/singleton-manager");
 /**
  * Pull metadata records from database to local files
  *
@@ -53,42 +54,76 @@ class Pull extends core_1.Command {
         filter: core_1.Flags.string({ description: 'Additional filter for pulling specific records' }),
         'dry-run': core_1.Flags.boolean({ description: 'Show what would be pulled without actually pulling' }),
         'multi-file': core_1.Flags.string({ description: 'Create a single file with multiple records (provide filename)' }),
+        verbose: core_1.Flags.boolean({ char: 'v', description: 'Show detailed output' }),
     };
     async run() {
         const { flags } = await this.parse(Pull);
         const spinner = (0, ora_classic_1.default)();
         try {
-            // Load MJ config
+            // Load MJ config first (before changing directory)
             spinner.start('Loading configuration');
             const mjConfig = (0, config_1.loadMJConfig)();
             if (!mjConfig) {
                 this.error('No mj.config.cjs found in current directory or parent directories');
             }
+            // Stop spinner before provider initialization (which logs to console)
+            spinner.stop();
             // Initialize data provider
             const provider = await (0, provider_utils_1.initializeProvider)(mjConfig);
-            // Initialize sync engine
-            const syncEngine = new sync_engine_1.SyncEngine((0, provider_utils_1.getSystemUser)());
-            await syncEngine.initialize();
-            spinner.succeed('Configuration loaded');
-            // Find entity directory
-            const entityDirs = await this.findEntityDirectories(flags.entity);
-            if (entityDirs.length === 0) {
-                this.error(`No directory found for entity "${flags.entity}". Run "mj-sync init" first.`);
-            }
+            // Get singleton sync engine
+            const syncEngine = await (0, singleton_manager_1.getSyncEngine)((0, provider_utils_1.getSystemUser)());
+            // Show success after all initialization is complete
+            spinner.succeed('Configuration and metadata loaded');
             let targetDir;
-            if (entityDirs.length === 1) {
-                targetDir = entityDirs[0];
+            let entityConfig;
+            // Check if we should use a specific target directory
+            const envTargetDir = process.env.METADATA_SYNC_TARGET_DIR;
+            if (envTargetDir) {
+                if (flags.verbose) {
+                    console.log(`Using specified target directory: ${envTargetDir}`);
+                }
+                process.chdir(envTargetDir);
+                targetDir = process.cwd();
+                // Load entity config from the current directory
+                entityConfig = await (0, config_1.loadEntityConfig)(targetDir);
+                if (!entityConfig) {
+                    this.error(`No .mj-sync.json found in ${targetDir}`);
+                }
+                if (entityConfig.entity !== flags.entity) {
+                    this.error(`Directory ${targetDir} is configured for entity "${entityConfig.entity}", not "${flags.entity}"`);
+                }
             }
             else {
-                // Multiple directories found, ask user
-                targetDir = await (0, prompts_1.select)({
-                    message: `Multiple directories found for entity "${flags.entity}". Which one to use?`,
-                    choices: entityDirs.map(dir => ({ name: dir, value: dir }))
-                });
+                // Original behavior - find entity directory
+                const entityDirs = await this.findEntityDirectories(flags.entity);
+                if (entityDirs.length === 0) {
+                    this.error(`No directory found for entity "${flags.entity}". Run "mj-sync init" first.`);
+                }
+                if (entityDirs.length === 1) {
+                    targetDir = entityDirs[0];
+                }
+                else {
+                    // Multiple directories found, ask user
+                    targetDir = await (0, prompts_1.select)({
+                        message: `Multiple directories found for entity "${flags.entity}". Which one to use?`,
+                        choices: entityDirs.map(dir => ({ name: dir, value: dir }))
+                    });
+                }
+                entityConfig = await (0, config_1.loadEntityConfig)(targetDir);
+                if (!entityConfig) {
+                    this.error(`Invalid entity configuration in ${targetDir}`);
+                }
             }
-            const entityConfig = await (0, config_1.loadEntityConfig)(targetDir);
-            if (!entityConfig) {
-                this.error(`Invalid entity configuration in ${targetDir}`);
+            // Show configuration notice only if relevant
+            if (entityConfig.pull?.appendRecordsToExistingFile && entityConfig.pull?.newFileName) {
+                const targetFile = path_1.default.join(targetDir, entityConfig.pull.newFileName.endsWith('.json')
+                    ? entityConfig.pull.newFileName
+                    : `${entityConfig.pull.newFileName}.json`);
+                if (await fs_extra_1.default.pathExists(targetFile)) {
+                    // File exists - inform about append behavior
+                    this.log(`\n📝 Configuration: New records will be appended to existing file '${path_1.default.basename(targetFile)}'`);
+                }
+                // If file doesn't exist, no need to mention anything special - we're just creating it
             }
             // Pull records
             spinner.start(`Pulling ${flags.entity} records`);
@@ -102,7 +137,8 @@ class Pull extends core_1.Command {
             }
             const result = await rv.RunView({
                 EntityName: flags.entity,
-                ExtraFilter: filter
+                ExtraFilter: filter,
+                ResultType: 'entity_object'
             }, (0, provider_utils_1.getSystemUser)());
             if (!result.Success) {
                 this.error(`Failed to pull records: ${result.ErrorMessage}`);
@@ -112,6 +148,39 @@ class Pull extends core_1.Command {
                 this.log(`\nDry run mode - would pull ${result.Results.length} records to ${targetDir}`);
                 return;
             }
+            // Check if we need to wait for async property loading
+            if (entityConfig.pull?.externalizeFields && result.Results.length > 0) {
+                const metadata = new core_2.Metadata();
+                const entityInfo = metadata.EntityByName(flags.entity);
+                if (entityInfo) {
+                    const externalizeConfig = entityConfig.pull.externalizeFields;
+                    let fieldsToExternalize = [];
+                    if (Array.isArray(externalizeConfig)) {
+                        if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {
+                            // Simple string array
+                            fieldsToExternalize = externalizeConfig;
+                        }
+                        else {
+                            // New pattern format
+                            fieldsToExternalize = externalizeConfig
+                                .map(item => item.field);
+                        }
+                    }
+                    else {
+                        // Object format
+                        fieldsToExternalize = Object.keys(externalizeConfig);
+                    }
+                    // Get all field names from entity metadata
+                    const metadataFieldNames = entityInfo.Fields.map(f => f.Name);
+                    // Check if any externalized fields are NOT in metadata (likely computed properties)
+                    const computedFields = fieldsToExternalize.filter(f => !metadataFieldNames.includes(f));
+                    if (computedFields.length > 0) {
+                        spinner.start(`Waiting 5 seconds for async property loading in ${flags.entity} (${computedFields.join(', ')})...`);
+                        await new Promise(resolve => setTimeout(resolve, 5000));
+                        spinner.succeed('Async property loading wait complete');
+                    }
+                }
+            }
             // Process each record
             const entityInfo = syncEngine.getEntityInfo(flags.entity);
             if (!entityInfo) {
@@ -119,6 +188,9 @@ class Pull extends core_1.Command {
             }
             spinner.start('Processing records');
             let processed = 0;
+            let updated = 0;
+            let created = 0;
+            let skipped = 0;
             // If multi-file flag is set, collect all records
             if (flags['multi-file']) {
                 const allRecords = [];
@@ -130,10 +202,12 @@ class Pull extends core_1.Command {
                             primaryKey[pk.Name] = record[pk.Name];
                         }
                         // Process record for multi-file
-                        const recordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine);
+                        const recordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine, flags, true);
                         allRecords.push(recordData);
                         processed++;
-                        spinner.text = `Processing records (${processed}/${result.Results.length})`;
+                        if (flags.verbose) {
+                            spinner.text = `Processing records (${processed}/${result.Results.length})`;
+                        }
                     }
                     catch (error) {
                         this.warn(`Failed to process record: ${error.message || error}`);
@@ -148,33 +222,213 @@ class Pull extends core_1.Command {
                 }
             }
             else {
-                // Original single-file-per-record logic
+                // Smart update logic for single-file-per-record
+                spinner.text = 'Scanning for existing files...';
+                // Find existing files
+                const filePattern = entityConfig.pull?.filePattern || entityConfig.filePattern || '*.json';
+                const existingFiles = await this.findExistingFiles(targetDir, filePattern);
+                if (flags.verbose) {
+                    this.log(`Found ${existingFiles.length} existing files matching pattern '${filePattern}'`);
+                    existingFiles.forEach(f => this.log(`  - ${path_1.default.basename(f)}`));
+                }
+                // Load existing records and build lookup map
+                const existingRecordsMap = await this.loadExistingRecords(existingFiles, entityInfo);
+                if (flags.verbose) {
+                    this.log(`Loaded ${existingRecordsMap.size} existing records from files`);
+                }
+                // Separate records into new and existing
+                const newRecords = [];
+                const existingRecordsToUpdate = [];
                 for (const record of result.Results) {
+                    // Build primary key
+                    const primaryKey = {};
+                    for (const pk of entityInfo.PrimaryKeys) {
+                        primaryKey[pk.Name] = record[pk.Name];
+                    }
+                    // Create lookup key
+                    const lookupKey = this.createPrimaryKeyLookup(primaryKey);
+                    const existingFileInfo = existingRecordsMap.get(lookupKey);
+                    if (existingFileInfo) {
+                        // Record exists locally
+                        if (entityConfig.pull?.updateExistingRecords !== false) {
+                            existingRecordsToUpdate.push({ record, primaryKey, filePath: existingFileInfo.filePath });
+                        }
+                        else {
+                            skipped++;
+                            if (flags.verbose) {
+                                this.log(`Skipping existing record: ${lookupKey}`);
+                            }
+                        }
+                    }
+                    else {
+                        // Record doesn't exist locally
+                        if (entityConfig.pull?.createNewFileIfNotFound !== false) {
+                            newRecords.push({ record, primaryKey });
+                        }
+                        else {
+                            skipped++;
+                            if (flags.verbose) {
+                                this.log(`Skipping new record (createNewFileIfNotFound=false): ${lookupKey}`);
+                            }
+                        }
+                    }
+                }
+                // Track which files have been backed up to avoid duplicates
+                const backedUpFiles = new Set();
+                // Process existing records updates
+                for (const { record, primaryKey, filePath } of existingRecordsToUpdate) {
                     try {
-                        // Build primary key
-                        const primaryKey = {};
-                        for (const pk of entityInfo.PrimaryKeys) {
-                            primaryKey[pk.Name] = record[pk.Name];
+                        spinner.text = `Updating existing records (${updated + 1}/${existingRecordsToUpdate.length})`;
+                        // Create backup if configured (only once per file)
+                        if (entityConfig.pull?.backupBeforeUpdate && !backedUpFiles.has(filePath)) {
+                            await this.createBackup(filePath, entityConfig.pull?.backupDirectory);
+                            backedUpFiles.add(filePath);
+                        }
+                        // Load existing file data
+                        const existingData = await fs_extra_1.default.readJson(filePath);
+                        // Find the specific existing record that matches this primary key
+                        let existingRecordData;
+                        if (Array.isArray(existingData)) {
+                            // Find the matching record in the array
+                            const matchingRecord = existingData.find(r => this.createPrimaryKeyLookup(r.primaryKey || {}) === this.createPrimaryKeyLookup(primaryKey));
+                            existingRecordData = matchingRecord || existingData[0]; // Fallback to first if not found
+                        }
+                        else {
+                            existingRecordData = existingData;
                         }
-                        // Process record
-                        await this.processRecord(record, primaryKey, targetDir, entityConfig, syncEngine);
+                        // Process the new record data (isNewRecord = false for updates)
+                        const newRecordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine, flags, false, existingRecordData);
+                        // Apply merge strategy
+                        const mergedData = await this.mergeRecords(existingRecordData, newRecordData, entityConfig.pull?.mergeStrategy || 'merge', entityConfig.pull?.preserveFields || []);
+                        // Write updated data
+                        if (Array.isArray(existingData)) {
+                            // Update the record in the array
+                            const index = existingData.findIndex(r => this.createPrimaryKeyLookup(r.primaryKey || {}) === this.createPrimaryKeyLookup(primaryKey));
+                            if (index >= 0) {
+                                existingData[index] = mergedData;
+                                await fs_extra_1.default.writeJson(filePath, existingData, { spaces: 2 });
+                            }
+                        }
+                        else {
+                            await fs_extra_1.default.writeJson(filePath, mergedData, { spaces: 2 });
+                        }
+                        updated++;
                         processed++;
-                        spinner.text = `Processing records (${processed}/${result.Results.length})`;
+                        if (flags.verbose) {
+                            this.log(`Updated: ${filePath}`);
+                        }
                     }
                     catch (error) {
-                        this.warn(`Failed to process record: ${error.message || error}`);
+                        this.warn(`Failed to update record: ${error.message || error}`);
                     }
                 }
-                spinner.succeed(`Pulled ${processed} records to ${targetDir}`);
+                // Process new records
+                if (newRecords.length > 0) {
+                    spinner.text = `Creating new records (0/${newRecords.length})`;
+                    if (entityConfig.pull?.appendRecordsToExistingFile && entityConfig.pull?.newFileName) {
+                        // Append all new records to a single file
+                        const fileName = entityConfig.pull.newFileName.endsWith('.json')
+                            ? entityConfig.pull.newFileName
+                            : `${entityConfig.pull.newFileName}.json`;
+                        const filePath = path_1.default.join(targetDir, fileName);
+                        // Load existing file if it exists
+                        let existingData = [];
+                        if (await fs_extra_1.default.pathExists(filePath)) {
+                            const fileData = await fs_extra_1.default.readJson(filePath);
+                            existingData = Array.isArray(fileData) ? fileData : [fileData];
+                        }
+                        // Process and append all new records
+                        for (const { record, primaryKey } of newRecords) {
+                            try {
+                                // For new records, pass isNewRecord = true (default)
+                                const recordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine, flags, true);
+                                existingData.push(recordData);
+                                created++;
+                                processed++;
+                                if (flags.verbose) {
+                                    spinner.text = `Creating new records (${created}/${newRecords.length})`;
+                                }
+                            }
+                            catch (error) {
+                                this.warn(`Failed to process new record: ${error.message || error}`);
+                            }
+                        }
+                        // Write the combined data
+                        await fs_extra_1.default.writeJson(filePath, existingData, { spaces: 2 });
+                        if (flags.verbose) {
+                            this.log(`Appended ${created} new records to: ${filePath}`);
+                        }
+                    }
+                    else {
+                        // Create individual files for each new record
+                        for (const { record, primaryKey } of newRecords) {
+                            try {
+                                await this.processRecord(record, primaryKey, targetDir, entityConfig, syncEngine, flags);
+                                created++;
+                                processed++;
+                                if (flags.verbose) {
+                                    spinner.text = `Creating new records (${created}/${newRecords.length})`;
+                                }
+                            }
+                            catch (error) {
+                                this.warn(`Failed to process new record: ${error.message || error}`);
+                            }
+                        }
+                    }
+                }
+                // Final status
+                const statusParts = [`Processed ${processed} records`];
+                if (updated > 0)
+                    statusParts.push(`updated ${updated}`);
+                if (created > 0)
+                    statusParts.push(`created ${created}`);
+                if (skipped > 0)
+                    statusParts.push(`skipped ${skipped}`);
+                spinner.succeed(statusParts.join(', '));
             }
         }
         catch (error) {
             spinner.fail('Pull failed');
+            // Enhanced error logging for debugging
+            this.log('\n=== Pull Error Details ===');
+            this.log(`Error type: ${error?.constructor?.name || 'Unknown'}`);
+            this.log(`Error message: ${error instanceof Error ? error.message : String(error)}`);
+            if (error instanceof Error && error.stack) {
+                this.log(`\nStack trace:`);
+                this.log(error.stack);
+            }
+            // Log context information
+            this.log(`\nContext:`);
+            this.log(`- Working directory: ${config_manager_1.configManager.getOriginalCwd()}`);
+            this.log(`- Entity: ${flags.entity || 'not specified'}`);
+            this.log(`- Filter: ${flags.filter || 'none'}`);
+            this.log(`- Flags: ${JSON.stringify(flags, null, 2)}`);
+            // Check if error is related to common issues
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (errorMessage.includes('No directory found for entity')) {
+                this.log(`\nHint: This appears to be an entity directory configuration issue.`);
+                this.log(`Run "mj-sync init" to create directories or ensure .mj-sync.json files exist.`);
+            }
+            else if (errorMessage.includes('database') || errorMessage.includes('connection')) {
+                this.log(`\nHint: This appears to be a database connectivity issue.`);
+                this.log(`Check your mj.config.cjs configuration and database connectivity.`);
+            }
+            else if (errorMessage.includes('Failed to pull records')) {
+                this.log(`\nHint: This appears to be a database query issue.`);
+                this.log(`Check if the entity name "${flags.entity}" is correct and exists in the database.`);
+            }
+            else if (errorMessage.includes('Entity information not found')) {
+                this.log(`\nHint: The entity "${flags.entity}" was not found in metadata.`);
+                this.log(`Check the entity name spelling and ensure it exists in the database.`);
+            }
             this.error(error);
         }
         finally {
-            // Clean up database connection
+            // Clean up database connection and reset singletons
             await (0, provider_utils_1.cleanupProvider)();
+            (0, singleton_manager_1.resetSyncEngine)();
+            // Exit process to prevent background MJ tasks from throwing errors
+            process.exit(0);
         }
     }
     /**
@@ -207,7 +461,7 @@ class Pull extends core_1.Command {
                 }
             }
         };
-        await searchDirs(process.cwd());
+        await searchDirs(config_manager_1.configManager.getOriginalCwd());
         return dirs;
     }
     /**
@@ -224,8 +478,8 @@ class Pull extends core_1.Command {
      * @returns Promise that resolves when file is written
      * @private
      */
-    async processRecord(record, primaryKey, targetDir, entityConfig, syncEngine) {
-        const recordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine);
+    async processRecord(record, primaryKey, targetDir, entityConfig, syncEngine, flags) {
+        const recordData = await this.processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine, flags, true);
         // Determine file path
         const fileName = this.buildFileName(primaryKey, entityConfig);
         const filePath = path_1.default.join(targetDir, fileName);
@@ -243,21 +497,56 @@ class Pull extends core_1.Command {
      * @param targetDir - Directory where files will be saved
      * @param entityConfig - Entity configuration with defaults and settings
      * @param syncEngine - Sync engine for checksum calculation
+     * @param flags - Command flags
+     * @param isNewRecord - Whether this is a new record
+     * @param existingRecordData - Existing record data to preserve field selection
+     * @param currentDepth - Current recursion depth for recursive entities
+     * @param ancestryPath - Set of IDs in current ancestry chain to prevent circular references
      * @returns Promise resolving to formatted RecordData
      * @private
      */
-    async processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine) {
-        // Build record data
-        const recordData = {
-            primaryKey: primaryKey,
-            fields: {},
-            sync: {
-                lastModified: new Date().toISOString(),
-                checksum: ''
+    async processRecordData(record, primaryKey, targetDir, entityConfig, syncEngine, flags, isNewRecord = true, existingRecordData, currentDepth = 0, ancestryPath = new Set()) {
+        // Build record data - we'll restructure at the end for proper ordering
+        const fields = {};
+        const relatedEntities = {};
+        // Debug: Log all fields in first record (only in verbose mode)
+        if (flags?.verbose) {
+            const recordKeys = Object.keys(record);
+            console.log('\n=== DEBUG: Processing record ===');
+            console.log('Entity:', entityConfig.entity);
+            console.log('Total fields:', recordKeys.length);
+            console.log('Field names:', recordKeys.filter(k => !k.startsWith('__mj_')).join(', '));
+            console.log('Has TemplateText?:', recordKeys.includes('TemplateText'));
+            console.log('externalizeFields config:', entityConfig.pull?.externalizeFields);
+        }
+        // Get the underlying data from the entity object
+        // If it's an entity object, it will have a GetAll() method
+        let dataToProcess = record;
+        if (typeof record.GetAll === 'function') {
+            // It's an entity object, get the underlying data
+            dataToProcess = record.GetAll();
+        }
+        // Get externalize configuration for pattern lookup
+        const externalizeConfig = entityConfig.pull?.externalizeFields;
+        let externalizeMap = new Map();
+        if (externalizeConfig) {
+            if (Array.isArray(externalizeConfig)) {
+                if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {
+                    // Simple string array
+                    externalizeConfig.forEach(f => externalizeMap.set(f, undefined));
+                }
+                else {
+                    // New pattern format
+                    externalizeConfig.forEach(item => externalizeMap.set(item.field, item.pattern));
+                }
             }
-        };
-        // Process fields
-        for (const [fieldName, fieldValue] of Object.entries(record)) {
+            else {
+                // Object format
+                Object.keys(externalizeConfig).forEach(f => externalizeMap.set(f, undefined));
+            }
+        }
+        // Process regular fields from the underlying data
+        for (const [fieldName, fieldValue] of Object.entries(dataToProcess)) {
             // Skip primary key fields
             if (primaryKey[fieldName] !== undefined) {
                 continue;
@@ -266,49 +555,312 @@ class Pull extends core_1.Command {
             if (fieldName.startsWith('__mj_')) {
                 continue;
             }
+            // Skip excluded fields
+            if (entityConfig.pull?.excludeFields?.includes(fieldName)) {
+                continue;
+            }
+            // Skip fields already externalized
+            if (fields[fieldName]) {
+                continue;
+            }
+            // Skip virtual/computed fields - check entity metadata
+            const metadata = new core_2.Metadata();
+            const entityInfo = metadata.EntityByName(entityConfig.entity);
+            if (entityInfo) {
+                const fieldInfo = entityInfo.Fields.find(f => f.Name === fieldName);
+                if (fieldInfo && !fieldInfo.IsVirtual) {
+                    // Field exists in metadata and is not virtual, keep it
+                }
+                else if (fieldInfo && fieldInfo.IsVirtual) {
+                    // Skip virtual fields
+                    continue;
+                }
+                else if (!fieldInfo) {
+                    // Field not in metadata at all
+                    // Check if it's explicitly configured for externalization, lookup, or exclusion
+                    const isConfiguredField = entityConfig.pull?.externalizeFields?.includes(fieldName) ||
+                        entityConfig.pull?.lookupFields?.[fieldName] ||
+                        entityConfig.pull?.excludeFields?.includes(fieldName);
+                    if (!isConfiguredField) {
+                        // Skip fields not in metadata and not explicitly configured
+                        continue;
+                    }
+                    // Otherwise, allow the field to be processed since it's explicitly configured
+                }
+            }
+            // Check if this field should be converted to a lookup
+            const lookupConfig = entityConfig.pull?.lookupFields?.[fieldName];
+            if (lookupConfig && fieldValue) {
+                // Convert foreign key to @lookup reference
+                const lookupValue = await this.convertToLookup(fieldValue, lookupConfig.entity, lookupConfig.field, syncEngine);
+                if (lookupValue) {
+                    fields[fieldName] = lookupValue;
+                    continue;
+                }
+            }
             // Check if this is an external file field
             if (await this.shouldExternalizeField(fieldName, fieldValue, entityConfig)) {
-                const fileName = await this.createExternalFile(targetDir, primaryKey, fieldName, String(fieldValue));
-                recordData.fields[fieldName] = `@file:${fileName}`;
+                // Check if this field is preserved and already has a @file: reference
+                const isPreservedField = entityConfig.pull?.preserveFields?.includes(fieldName);
+                const existingFieldValue = existingRecordData?.fields?.[fieldName];
+                if (isPreservedField && existingFieldValue && typeof existingFieldValue === 'string' && existingFieldValue.startsWith('@file:')) {
+                    // Field is preserved and has existing @file: reference - update the existing file
+                    const existingFilePath = existingFieldValue.replace('@file:', '');
+                    const fullPath = path_1.default.join(targetDir, existingFilePath);
+                    // Ensure directory exists
+                    await fs_extra_1.default.ensureDir(path_1.default.dirname(fullPath));
+                    // Write the content to the existing file path
+                    await fs_extra_1.default.writeFile(fullPath, String(fieldValue), 'utf-8');
+                    // Keep the existing @file: reference
+                    fields[fieldName] = existingFieldValue;
+                }
+                else {
+                    // Normal externalization - create new file
+                    const pattern = externalizeMap.get(fieldName);
+                    const fileName = await this.createExternalFile(targetDir, record, primaryKey, fieldName, String(fieldValue), entityConfig, pattern);
+                    fields[fieldName] = fileName; // fileName already includes @file: prefix if pattern-based
+                }
             }
             else {
-                recordData.fields[fieldName] = fieldValue;
+                fields[fieldName] = fieldValue;
+            }
+        }
+        // Now check for externalized fields that might be computed properties
+        // We process ALL externalized fields, including those not in the data
+        if (entityConfig.pull?.externalizeFields && typeof record.GetAll === 'function') {
+            const externalizeConfig = entityConfig.pull.externalizeFields;
+            // Normalize configuration to array format
+            let externalizeItems = [];
+            if (Array.isArray(externalizeConfig)) {
+                if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {
+                    // Simple string array
+                    externalizeItems = externalizeConfig.map(f => ({ field: f }));
+                }
+                else {
+                    // Already in the new format
+                    externalizeItems = externalizeConfig;
+                }
+            }
+            else {
+                // Object format
+                externalizeItems = Object.entries(externalizeConfig).map(([field, config]) => ({
+                    field,
+                    pattern: undefined // Will use default pattern
+                }));
+            }
+            // Get the keys from the underlying data to identify computed properties
+            const dataKeys = Object.keys(dataToProcess);
+            for (const externalItem of externalizeItems) {
+                const externalField = externalItem.field;
+                // Only process fields that are NOT in the underlying data
+                // (these are likely computed properties)
+                if (dataKeys.includes(externalField)) {
+                    continue; // This was already processed in the main loop
+                }
+                try {
+                    // Use bracket notation to access properties (including getters)
+                    const fieldValue = record[externalField];
+                    if (fieldValue !== undefined && fieldValue !== null && fieldValue !== '') {
+                        if (await this.shouldExternalizeField(externalField, fieldValue, entityConfig)) {
+                            // Check if this field is preserved and already has a @file: reference
+                            const isPreservedField = entityConfig.pull?.preserveFields?.includes(externalField);
+                            const existingFieldValue = existingRecordData?.fields?.[externalField];
+                            if (isPreservedField && existingFieldValue && typeof existingFieldValue === 'string' && existingFieldValue.startsWith('@file:')) {
+                                // Field is preserved and has existing @file: reference - update the existing file
+                                const existingFilePath = existingFieldValue.replace('@file:', '');
+                                const fullPath = path_1.default.join(targetDir, existingFilePath);
+                                // Ensure directory exists
+                                await fs_extra_1.default.ensureDir(path_1.default.dirname(fullPath));
+                                // Write the content to the existing file path
+                                await fs_extra_1.default.writeFile(fullPath, String(fieldValue), 'utf-8');
+                                // Keep the existing @file: reference
+                                fields[externalField] = existingFieldValue;
+                            }
+                            else {
+                                // Normal externalization - create new file
+                                const fileName = await this.createExternalFile(targetDir, record, primaryKey, externalField, String(fieldValue), entityConfig, externalItem.pattern);
+                                fields[externalField] = fileName; // fileName already includes @file: prefix if pattern-based
+                            }
+                        }
+                        else {
+                            // Include the field value if not externalized
+                            fields[externalField] = fieldValue;
+                        }
+                    }
+                }
+                catch (error) {
+                    // Property might not exist, that's okay
+                    if (flags?.verbose) {
+                        console.log(`Could not get property ${externalField}: ${error}`);
+                    }
+                }
             }
         }
         // Pull related entities if configured
         if (entityConfig.pull?.relatedEntities) {
-            recordData.relatedEntities = await this.pullRelatedEntities(record, entityConfig.pull.relatedEntities, syncEngine);
+            const related = await this.pullRelatedEntities(record, entityConfig.pull.relatedEntities, syncEngine, entityConfig, flags, currentDepth, ancestryPath);
+            Object.assign(relatedEntities, related);
+        }
+        // Get entity metadata to check defaults
+        const metadata = new core_2.Metadata();
+        const entityInfo = metadata.EntityByName(entityConfig.entity);
+        // Filter out null values and fields matching their defaults
+        const cleanedFields = {};
+        // Get the set of fields that existed in the original record (if updating)
+        const existingFieldNames = existingRecordData?.fields ? new Set(Object.keys(existingRecordData.fields)) : new Set();
+        for (const [fieldName, fieldValue] of Object.entries(fields)) {
+            let includeField = false;
+            if (!isNewRecord && existingFieldNames.has(fieldName)) {
+                // For updates: Always preserve fields that existed in the original record
+                includeField = true;
+            }
+            else {
+                // For new records or new fields in existing records:
+                // Skip null/undefined/empty string values
+                if (fieldValue === null || fieldValue === undefined || fieldValue === '') {
+                    includeField = false;
+                }
+                else if (entityInfo) {
+                    // Check if value matches the field's default
+                    const fieldInfo = entityInfo.Fields.find(f => f.Name === fieldName);
+                    if (fieldInfo && fieldInfo.DefaultValue !== null && fieldInfo.DefaultValue !== undefined) {
+                        // Compare with default value
+                        if (fieldValue === fieldInfo.DefaultValue) {
+                            includeField = false;
+                        }
+                        // Special handling for boolean defaults (might be stored as strings)
+                        else if (typeof fieldValue === 'boolean' &&
+                            (fieldInfo.DefaultValue === (fieldValue ? '1' : '0') ||
+                                fieldInfo.DefaultValue === (fieldValue ? 'true' : 'false'))) {
+                            includeField = false;
+                        }
+                        // Special handling for numeric defaults that might be strings
+                        else if (typeof fieldValue === 'number' && String(fieldValue) === String(fieldInfo.DefaultValue)) {
+                            includeField = false;
+                        }
+                        else {
+                            includeField = true;
+                        }
+                    }
+                    else {
+                        // No default value defined, include if not null/empty
+                        includeField = true;
+                    }
+                }
+                else {
+                    // No entity info, include if not null/empty
+                    includeField = true;
+                }
+            }
+            if (includeField) {
+                cleanedFields[fieldName] = fieldValue;
+            }
+        }
+        // Calculate checksum on cleaned fields
+        const checksum = syncEngine.calculateChecksum(cleanedFields);
+        // Build the final record data with proper ordering
+        // Use a new object to ensure property order
+        const recordData = {};
+        // 1. User fields first
+        recordData.fields = cleanedFields;
+        // 2. Related entities (if any)
+        if (Object.keys(relatedEntities).length > 0) {
+            recordData.relatedEntities = relatedEntities;
         }
-        // Calculate checksum
-        recordData.sync.checksum = syncEngine.calculateChecksum(recordData.fields);
+        // 3. Primary key (system field)
+        recordData.primaryKey = primaryKey;
+        // 4. Sync metadata (system field)
+        recordData.sync = {
+            lastModified: new Date().toISOString(),
+            checksum: checksum
+        };
         return recordData;
     }
+    /**
+     * Convert a foreign key value to a @lookup reference
+     *
+     * Looks up the related record and creates a @lookup string that can be
+     * resolved during push operations.
+     *
+     * @param foreignKeyValue - The foreign key value (ID)
+     * @param targetEntity - Name of the target entity
+     * @param targetField - Field in target entity to use for lookup
+     * @param syncEngine - Sync engine instance
+     * @returns @lookup string or null if lookup fails
+     * @private
+     */
+    async convertToLookup(foreignKeyValue, targetEntity, targetField, syncEngine) {
+        try {
+            // Get the related record
+            const metadata = new core_2.Metadata();
+            const targetEntityInfo = metadata.EntityByName(targetEntity);
+            if (!targetEntityInfo) {
+                this.warn(`Could not find entity ${targetEntity} for lookup`);
+                return null;
+            }
+            // Load the related record
+            const primaryKeyField = targetEntityInfo.PrimaryKeys?.[0]?.Name || 'ID';
+            const rv = new core_2.RunView();
+            const result = await rv.RunView({
+                EntityName: targetEntity,
+                ExtraFilter: `${primaryKeyField} = '${String(foreignKeyValue).replace(/'/g, "''")}'`,
+                ResultType: 'entity_object'
+            }, (0, provider_utils_1.getSystemUser)());
+            if (!result.Success || result.Results.length === 0) {
+                this.warn(`Could not find ${targetEntity} with ${primaryKeyField} = ${foreignKeyValue}`);
+                return null;
+            }
+            const relatedRecord = result.Results[0];
+            const lookupValue = relatedRecord[targetField];
+            if (!lookupValue) {
+                this.warn(`${targetEntity} record missing ${targetField} field`);
+                return null;
+            }
+            // Return the @lookup reference
+            return `@lookup:${targetEntity}.${targetField}=${lookupValue}`;
+        }
+        catch (error) {
+            this.warn(`Failed to create lookup for ${targetEntity}: ${error}`);
+            return null;
+        }
+    }
     /**
      * Determine if a field should be saved to an external file
      *
-     * Checks if a field contains substantial text content that would be better
-     * stored in a separate file rather than inline in the JSON. Uses heuristics
-     * based on field name and content length.
+     * Checks if a field is configured for externalization or contains substantial
+     * text content that would be better stored in a separate file.
      *
      * @param fieldName - Name of the field to check
      * @param fieldValue - Value of the field
-     * @param entityConfig - Entity configuration (for future extension)
+     * @param entityConfig - Entity configuration with externalization settings
      * @returns Promise resolving to true if field should be externalized
      * @private
      */
     async shouldExternalizeField(fieldName, fieldValue, entityConfig) {
-        // Only externalize string fields with significant content
+        // Only externalize string fields
         if (typeof fieldValue !== 'string') {
             return false;
         }
-        // Check if it's a known large text field
-        const largeTextFields = ['Prompt', 'Template', 'Notes', 'Description',
-            'Content', 'Body', 'Text', 'HTML', 'SQL'];
-        if (largeTextFields.some(f => fieldName.toLowerCase().includes(f.toLowerCase()))) {
-            // Only externalize if content is substantial (more than 100 chars or has newlines)
-            return fieldValue.length > 100 || fieldValue.includes('\n');
+        // Check if field is configured for externalization
+        const externalizeConfig = entityConfig.pull?.externalizeFields;
+        if (!externalizeConfig) {
+            return false;
+        }
+        if (Array.isArray(externalizeConfig)) {
+            if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {
+                // Simple string array
+                return externalizeConfig.includes(fieldName);
+            }
+            else {
+                // New pattern format
+                return externalizeConfig
+                    .some(item => item.field === fieldName);
+            }
+        }
+        else {
+            // Object format
+            return fieldName in externalizeConfig;
         }
-        return false;
     }
     /**
      * Create an external file for a field value
@@ -316,36 +868,103 @@ class Pull extends core_1.Command {
      * Saves large text content to a separate file and returns the filename.
      * Automatically determines appropriate file extension based on field name
      * and content type (e.g., .md for prompts, .html for templates).
+     * Uses the entity's name field for the filename if available.
      *
      * @param targetDir - Directory to save the file
-     * @param primaryKey - Primary key for filename generation
+     * @param record - Full record to extract name field from
+     * @param primaryKey - Primary key for filename generation fallback
      * @param fieldName - Name of the field being externalized
      * @param content - Content to write to the file
+     * @param entityConfig - Entity configuration
      * @returns Promise resolving to the created filename
      * @private
      */
-    async createExternalFile(targetDir, primaryKey, fieldName, content) {
-        // Determine file extension based on field name and content
-        let extension = '.txt';
-        if (fieldName.toLowerCase().includes('prompt')) {
-            extension = '.md';
-        }
-        else if (fieldName.toLowerCase().includes('template')) {
-            if (content.includes('<html') || content.includes('<!DOCTYPE')) {
-                extension = '.html';
+    async createExternalFile(targetDir, record, primaryKey, fieldName, content, entityConfig, pattern) {
+        // If pattern is provided, use it to generate the full path
+        if (pattern) {
+            // Replace placeholders in the pattern
+            let resolvedPattern = pattern;
+            // Get entity metadata for field lookups
+            const metadata = new core_2.Metadata();
+            const entityInfo = metadata.EntityByName(entityConfig.entity);
+            // Replace {Name} with the entity's name field value
+            if (entityInfo) {
+                const nameField = entityInfo.Fields.find(f => f.IsNameField);
+                if (nameField && record[nameField.Name]) {
+                    const nameValue = String(record[nameField.Name])
+                        .replace(/[^a-zA-Z0-9\-_ ]/g, '') // Remove disallowed characters
+                        .replace(/\s+/g, '-') // Replace spaces with -
+                        .toLowerCase(); // Make lowercase
+                    resolvedPattern = resolvedPattern.replace(/{Name}/g, nameValue);
+                }
+            }
+            // Replace {ID} with the primary key
+            const idValue = primaryKey.ID || Object.values(primaryKey)[0];
+            if (idValue) {
+                resolvedPattern = resolvedPattern.replace(/{ID}/g, String(idValue).toLowerCase());
             }
-            else if (content.includes('{{') || content.includes('{%')) {
-                extension = '.liquid';
+            // Replace {FieldName} with the current field name
+            resolvedPattern = resolvedPattern.replace(/{FieldName}/g, fieldName.toLowerCase());
+            // Replace any other {field} placeholders with field values from the record
+            const placeholderRegex = /{(\w+)}/g;
+            resolvedPattern = resolvedPattern.replace(placeholderRegex, (match, fieldName) => {
+                const value = record[fieldName];
+                if (value !== undefined && value !== null) {
+                    return String(value)
+                        .replace(/[^a-zA-Z0-9\-_ ]/g, '')
+                        .replace(/\s+/g, '-')
+                        .toLowerCase();
+                }
+                return match; // Keep placeholder if field not found
+            });
+            // Extract the file path from the pattern
+            const filePath = path_1.default.join(targetDir, resolvedPattern.replace('@file:', ''));
+            // Ensure directory exists
+            await fs_extra_1.default.ensureDir(path_1.default.dirname(filePath));
+            // Write the file
+            await fs_extra_1.default.writeFile(filePath, content, 'utf-8');
+            // Return the pattern as-is (it includes @file: prefix)
+            return resolvedPattern;
+        }
+        // Original logic for non-pattern based externalization
+        let extension = '.md'; // default to markdown
+        const externalizeConfig = entityConfig.pull?.externalizeFields;
+        if (externalizeConfig && !Array.isArray(externalizeConfig) && externalizeConfig[fieldName]?.extension) {
+            extension = externalizeConfig[fieldName].extension;
+            // Ensure extension starts with a dot
+            if (!extension.startsWith('.')) {
+                extension = '.' + extension;
             }
         }
-        else if (fieldName.toLowerCase().includes('sql')) {
-            extension = '.sql';
+        // Try to use the entity's name field for the filename
+        let baseFileName;
+        // Get entity metadata to find the name field
+        const metadata = new core_2.Metadata();
+        const entityInfo = metadata.EntityByName(entityConfig.entity);
+        if (entityInfo) {
+            // Find the name field
+            const nameField = entityInfo.Fields.find(f => f.IsNameField);
+            if (nameField && record[nameField.Name]) {
+                // Use the name field value, sanitized for filesystem
+                const nameValue = String(record[nameField.Name]);
+                // Remove disallowed characters (don't replace with _), replace spaces with -, and lowercase
+                baseFileName = nameValue
+                    .replace(/[^a-zA-Z0-9\-_ ]/g, '') // Remove disallowed characters
+                    .replace(/\s+/g, '-') // Replace spaces with -
+                    .toLowerCase(); // Make lowercase
+            }
+            else {
+                // Fallback to primary key
+                baseFileName = this.buildFileName(primaryKey, null).replace('.json', '');
+            }
         }
-        else if (fieldName.toLowerCase().includes('notes') || fieldName.toLowerCase().includes('description')) {
-            extension = '.md';
+        else {
+            // Fallback to primary key
+            baseFileName = this.buildFileName(primaryKey, null).replace('.json', '');
         }
-        const baseFileName = this.buildFileName(primaryKey, null).replace('.json', '');
-        const fileName = `${baseFileName}.${fieldName.toLowerCase()}${extension}`;
+        // Remove dot prefix from baseFileName if it exists (it will be a dot-prefixed name from buildFileName)
+        const cleanBaseFileName = baseFileName.startsWith('.') ? baseFileName.substring(1) : baseFileName;
+        const fileName = `.${cleanBaseFileName}.${fieldName.toLowerCase()}${extension}`;
         const filePath = path_1.default.join(targetDir, fileName);
         await fs_extra_1.default.writeFile(filePath, content, 'utf-8');
         return fileName;
@@ -356,6 +975,7 @@ class Pull extends core_1.Command {
      * Creates a safe filename based on the entity's primary key values.
      * Handles GUIDs by using first 8 characters, sanitizes special characters,
      * and creates composite names for multi-field keys.
+     * Files are prefixed with a dot to follow the metadata file convention.
      *
      * @param primaryKey - Primary key fields and values
      * @param entityConfig - Entity configuration (for future extension)
@@ -369,16 +989,16 @@ class Pull extends core_1.Command {
             // Single string key - use as base if it's a guid
             const key = keys[0];
             if (key.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)) {
-                // It's a GUID, use first 8 chars
-                return `${key.substring(0, 8)}.json`;
+                // It's a GUID, use first 8 chars, prefixed with dot, lowercase
+                return `.${key.substring(0, 8).toLowerCase()}.json`;
             }
-            // Use the whole key if not too long
+            // Use the whole key if not too long, prefixed with dot
             if (key.length <= 50) {
-                return `${key.replace(/[^a-zA-Z0-9-_]/g, '_')}.json`;
+                return `.${key.replace(/[^a-zA-Z0-9\-_]/g, '').toLowerCase()}.json`;
             }
         }
-        // Multiple keys or numeric - create composite name
-        return keys.map(k => String(k).replace(/[^a-zA-Z0-9-_]/g, '_')).join('-') + '.json';
+        // Multiple keys or numeric - create composite name, prefixed with dot
+        return '.' + keys.map(k => String(k).replace(/[^a-zA-Z0-9\-_]/g, '').toLowerCase()).join('-') + '.json';
     }
     /**
      * Pull related entities for a parent record
@@ -386,23 +1006,38 @@ class Pull extends core_1.Command {
      * Retrieves child records that have foreign key relationships to the parent.
      * Converts foreign key values to @parent references and supports nested
      * related entities for deep object graphs.
+     * NEW: Supports automatic recursive patterns for self-referencing entities.
      *
      * @param parentRecord - Parent entity record
      * @param relatedConfig - Configuration for related entities to pull
      * @param syncEngine - Sync engine instance
+     * @param entityConfig - Entity configuration
+     * @param flags - Command flags
+     * @param currentDepth - Current recursion depth for recursive entities
+     * @param ancestryPath - Set of IDs in current ancestry chain to prevent circular references
      * @returns Promise resolving to map of entity names to related records
      * @private
      */
-    async pullRelatedEntities(parentRecord, relatedConfig, syncEngine) {
+    async pullRelatedEntities(parentRecord, relatedConfig, syncEngine, entityConfig, flags, currentDepth = 0, ancestryPath = new Set()) {
         const relatedEntities = {};
         for (const [key, config] of Object.entries(relatedConfig)) {
             try {
-                // Get the parent's primary key value
-                const parentKeyValue = parentRecord[config.foreignKey];
+                // Get entity metadata to find primary key
+                const metadata = new core_2.Metadata();
+                const parentEntity = metadata.EntityByName(entityConfig.entity);
+                if (!parentEntity) {
+                    this.warn(`Could not find entity metadata for ${entityConfig.entity}`);
+                    continue;
+                }
+                // Get the parent's primary key value (usually ID)
+                const primaryKeyField = parentEntity.PrimaryKeys?.[0]?.Name || 'ID';
+                const parentKeyValue = parentRecord[primaryKeyField];
                 if (!parentKeyValue) {
-                    continue; // Skip if parent doesn't have the foreign key field
+                    this.warn(`Parent record missing primary key field ${primaryKeyField}`);
+                    continue;
                 }
                 // Build filter for related records
+                // The foreignKey is the field in the CHILD entity that points to this parent
                 let filter = `${config.foreignKey} = '${String(parentKeyValue).replace(/'/g, "''")}'`;
                 if (config.filter) {
                     filter += ` AND (${config.filter})`;
@@ -411,39 +1046,115 @@ class Pull extends core_1.Command {
                 const rv = new core_2.RunView();
                 const result = await rv.RunView({
                     EntityName: config.entity,
-                    ExtraFilter: filter
+                    ExtraFilter: filter,
+                    ResultType: 'entity_object'
                 }, (0, provider_utils_1.getSystemUser)());
                 if (!result.Success) {
                     this.warn(`Failed to pull related ${config.entity}: ${result.ErrorMessage}`);
                     continue;
                 }
+                // Get child entity metadata
+                const childEntity = metadata.EntityByName(config.entity);
+                if (!childEntity) {
+                    this.warn(`Could not find entity metadata for ${config.entity}`);
+                    continue;
+                }
+                // Check if we need to wait for async property loading for related entities
+                if (config.externalizeFields && result.Results.length > 0) {
+                    let fieldsToExternalize = [];
+                    if (Array.isArray(config.externalizeFields)) {
+                        if (config.externalizeFields.length > 0 && typeof config.externalizeFields[0] === 'string') {
+                            // Simple string array
+                            fieldsToExternalize = config.externalizeFields;
+                        }
+                        else {
+                            // New pattern format
+                            fieldsToExternalize = config.externalizeFields
+                                .map(item => item.field);
+                        }
+                    }
+                    else {
+                        // Object format
+                        fieldsToExternalize = Object.keys(config.externalizeFields);
+                    }
+                    // Get all field names from entity metadata
+                    const metadataFieldNames = childEntity.Fields.map(f => f.Name);
+                    // Check if any externalized fields are NOT in metadata (likely computed properties)
+                    const computedFields = fieldsToExternalize.filter(f => !metadataFieldNames.includes(f));
+                    if (computedFields.length > 0) {
+                        console.log(`Waiting 5 seconds for async property loading in related entity ${config.entity} (${computedFields.join(', ')})...`);
+                        await new Promise(resolve => setTimeout(resolve, 5000));
+                    }
+                }
                 // Process each related record
                 const relatedRecords = [];
+                if (flags?.verbose && result.Results.length > 0) {
+                    this.log(`Found ${result.Results.length} related ${config.entity} records at depth ${currentDepth}`);
+                }
                 for (const relatedRecord of result.Results) {
-                    const recordData = {
-                        fields: {}
-                    };
-                    // Process fields, omitting the foreign key since it will be set via @parent
-                    for (const [fieldName, fieldValue] of Object.entries(relatedRecord)) {
-                        // Skip internal fields
-                        if (fieldName.startsWith('__mj_')) {
-                            continue;
+                    // Build primary key for the related record
+                    const relatedPrimaryKey = {};
+                    for (const pk of childEntity.PrimaryKeys) {
+                        relatedPrimaryKey[pk.Name] = relatedRecord[pk.Name];
+                    }
+                    // Check for circular references in the current ancestry path
+                    const recordId = String(relatedPrimaryKey[childEntity.PrimaryKeys[0]?.Name || 'ID']);
+                    if (config.recursive && ancestryPath.has(recordId)) {
+                        if (flags?.verbose) {
+                            this.log(`Skipping circular reference for ${config.entity} with ID: ${recordId} (detected in ancestry path)`);
+                        }
+                        continue;
+                    }
+                    // Create new ancestry path for this branch (only track current hierarchy chain)
+                    const newAncestryPath = new Set(ancestryPath);
+                    if (config.recursive) {
+                        newAncestryPath.add(recordId);
+                    }
+                    // Determine related entities configuration for recursion
+                    let childRelatedConfig = config.relatedEntities;
+                    // If recursive is enabled, continue recursive fetching at child level
+                    if (config.recursive) {
+                        const maxDepth = config.maxDepth || 10;
+                        if (currentDepth < maxDepth) {
+                            // Create recursive configuration that references the same entity
+                            childRelatedConfig = {
+                                [key]: {
+                                    ...config,
+                                    // Keep same configuration but increment depth internally
+                                }
+                            };
+                            if (flags?.verbose) {
+                                this.log(`Processing recursive level ${currentDepth + 1} for ${config.entity} record ${recordId}`);
+                            }
                         }
-                        // Convert foreign key reference to @parent
-                        if (fieldName === config.foreignKey) {
-                            const parentFieldName = this.findParentField(parentRecord, parentKeyValue);
-                            if (parentFieldName) {
-                                recordData.fields[fieldName] = `@parent:${parentFieldName}`;
+                        else {
+                            // At max depth, don't recurse further
+                            childRelatedConfig = undefined;
+                            if (flags?.verbose) {
+                                this.log(`Max depth ${maxDepth} reached for recursive entity ${config.entity} at record ${recordId}`);
                             }
-                            continue;
                         }
-                        recordData.fields[fieldName] = fieldValue;
                     }
-                    // Pull nested related entities if configured
-                    if (config.relatedEntities) {
-                        recordData.relatedEntities = await this.pullRelatedEntities(relatedRecord, config.relatedEntities, syncEngine);
+                    // Process the related record using the same logic as parent records
+                    const relatedData = await this.processRecordData(relatedRecord, relatedPrimaryKey, '', // Not used for related entities since we don't externalize their fields
+                    {
+                        entity: config.entity,
+                        pull: {
+                            excludeFields: config.excludeFields || entityConfig.pull?.excludeFields,
+                            lookupFields: config.lookupFields || entityConfig.pull?.lookupFields,
+                            externalizeFields: config.externalizeFields,
+                            relatedEntities: childRelatedConfig
+                        }
+                    }, syncEngine, flags, true, // isNewRecord
+                    undefined, // existingRecordData
+                    currentDepth + 1, newAncestryPath);
+                    // Convert foreign key reference to @parent
+                    if (relatedData.fields[config.foreignKey]) {
+                        relatedData.fields[config.foreignKey] = `@parent:${primaryKeyField}`;
                     }
-                    relatedRecords.push(recordData);
+                    // The processRecordData method already filters nulls and defaults
+                    // No need to do it again here
+                    relatedRecords.push(relatedData);
                 }
                 if (relatedRecords.length > 0) {
                     relatedEntities[key] = relatedRecords;
@@ -477,6 +1188,189 @@ class Pull extends core_1.Command {
         }
         return null;
     }
+    /**
+     * Find existing files in a directory matching a pattern
+     *
+     * Searches for files that match the configured file pattern, used to identify
+     * which records already exist locally for smart update functionality.
+     *
+     * @param dir - Directory to search in
+     * @param pattern - Glob pattern to match files (e.g., "*.json")
+     * @returns Promise resolving to array of file paths
+     * @private
+     */
+    async findExistingFiles(dir, pattern) {
+        const files = [];
+        try {
+            const entries = await fs_extra_1.default.readdir(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isFile()) {
+                    const fileName = entry.name;
+                    // Simple pattern matching - could be enhanced with proper glob support
+                    if (pattern === '*.json' && fileName.endsWith('.json')) {
+                        files.push(path_1.default.join(dir, fileName));
+                    }
+                    else if (pattern === '.*.json' && fileName.startsWith('.') && fileName.endsWith('.json')) {
+                        // Handle dot-prefixed JSON files
+                        files.push(path_1.default.join(dir, fileName));
+                    }
+                    else if (pattern === fileName) {
+                        files.push(path_1.default.join(dir, fileName));
+                    }
+                    // TODO: Add more sophisticated glob pattern matching if needed
+                }
+            }
+        }
+        catch (error) {
+            // Directory might not exist yet
+            if (error.code !== 'ENOENT') {
+                throw error;
+            }
+        }
+        return files;
+    }
+    /**
+     * Load existing records from files and build a lookup map
+     *
+     * Reads all existing files and creates a map from primary key to file location,
+     * enabling efficient lookup during the update process.
+     *
+     * @param files - Array of file paths to load
+     * @param entityInfo - Entity metadata for primary key information
+     * @returns Map from primary key string to file info
+     * @private
+     */
+    async loadExistingRecords(files, entityInfo) {
+        const recordsMap = new Map();
+        for (const filePath of files) {
+            try {
+                const fileData = await fs_extra_1.default.readJson(filePath);
+                const records = Array.isArray(fileData) ? fileData : [fileData];
+                for (const record of records) {
+                    if (record.primaryKey) {
+                        const lookupKey = this.createPrimaryKeyLookup(record.primaryKey);
+                        recordsMap.set(lookupKey, { filePath, recordData: record });
+                    }
+                }
+            }
+            catch (error) {
+                // Skip files that can't be parsed
+                this.warn(`Could not load file ${filePath}: ${error}`);
+            }
+        }
+        return recordsMap;
+    }
+    /**
+     * Create a string lookup key from primary key values
+     *
+     * Generates a consistent string representation of primary key values
+     * for use in maps and comparisons.
+     *
+     * @param primaryKey - Primary key field names and values
+     * @returns String representation of the primary key
+     * @private
+     */
+    createPrimaryKeyLookup(primaryKey) {
+        const keys = Object.keys(primaryKey).sort();
+        return keys.map(k => `${k}:${primaryKey[k]}`).join('|');
+    }
+    /**
+     * Merge two record data objects based on configured strategy
+     *
+     * Combines existing and new record data according to the merge strategy:
+     * - 'overwrite': Replace all fields with new values
+     * - 'merge': Combine fields, with new values taking precedence
+     * - 'skip': Keep existing record unchanged
+     *
+     * @param existing - Existing record data
+     * @param newData - New record data from database
+     * @param strategy - Merge strategy to apply
+     * @param preserveFields - Field names that should never be overwritten
+     * @returns Merged record data
+     * @private
+     */
+    async mergeRecords(existing, newData, strategy, preserveFields) {
+        if (strategy === 'skip') {
+            return existing;
+        }
+        if (strategy === 'overwrite') {
+            // Build with proper ordering
+            const result = {};
+            // 1. Fields first
+            result.fields = { ...newData.fields };
+            // Restore preserved fields from existing
+            if (preserveFields.length > 0 && existing.fields) {
+                for (const field of preserveFields) {
+                    if (field in existing.fields) {
+                        result.fields[field] = existing.fields[field];
+                    }
+                }
+            }
+            // 2. Related entities (if any)
+            if (newData.relatedEntities) {
+                result.relatedEntities = newData.relatedEntities;
+            }
+            // 3. Primary key
+            result.primaryKey = newData.primaryKey;
+            // 4. Sync metadata
+            result.sync = newData.sync;
+            return result;
+        }
+        // Default 'merge' strategy
+        // Build with proper ordering
+        const result = {};
+        // 1. Fields first
+        result.fields = { ...existing.fields, ...newData.fields };
+        // Restore preserved fields
+        if (preserveFields.length > 0 && existing.fields) {
+            for (const field of preserveFields) {
+                if (field in existing.fields) {
+                    result.fields[field] = existing.fields[field];
+                }
+            }
+        }
+        // 2. Related entities (if any)
+        if (existing.relatedEntities || newData.relatedEntities) {
+            result.relatedEntities = {
+                ...existing.relatedEntities,
+                ...newData.relatedEntities
+            };
+        }
+        // 3. Primary key
+        result.primaryKey = newData.primaryKey || existing.primaryKey;
+        // 4. Sync metadata
+        result.sync = newData.sync;
+        return result;
+    }
+    /**
+     * Create a backup of a file before updating
+     *
+     * Creates a timestamped backup copy of the file in a backup directory
+     * with the original filename, timestamp suffix, and .backup extension.
+     * The backup directory defaults to .backups but can be configured.
+     *
+     * @param filePath - Path to the file to backup
+     * @param backupDirName - Name of the backup directory (optional)
+     * @returns Promise that resolves when backup is created
+     * @private
+     */
+    async createBackup(filePath, backupDirName) {
+        const dir = path_1.default.dirname(filePath);
+        const fileName = path_1.default.basename(filePath);
+        const backupDir = path_1.default.join(dir, backupDirName || '.backups');
+        // Ensure backup directory exists
+        await fs_extra_1.default.ensureDir(backupDir);
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        // Remove .json extension, add timestamp, then add .backup extension
+        const backupFileName = fileName.replace(/\.json$/, `.${timestamp}.backup`);
+        const backupPath = path_1.default.join(backupDir, backupFileName);
+        try {
+            await fs_extra_1.default.copy(filePath, backupPath);
+        }
+        catch (error) {
+            this.warn(`Could not create backup of ${filePath}: ${error}`);
+        }
+    }
 }
 exports.default = Pull;
 //# sourceMappingURL=index.js.map