@memberjunction/metadata-sync 2.51.0 → 2.53.0

package/dist/commands/push/index.js

@@ -38,11 +38,14 @@ const provider_utils_1 = require("../../lib/provider-utils");
 const core_2 = require("@memberjunction/core");
 const config_manager_1 = require("../../lib/config-manager");
 const singleton_manager_1 = require("../../lib/singleton-manager");
+const sqlserver_dataprovider_1 = require("@memberjunction/sqlserver-dataprovider");
 const global_1 = require("@memberjunction/global");
+const file_backup_manager_1 = require("../../lib/file-backup-manager");
 class Push extends core_1.Command {
     static description = 'Push local file changes to the database';
     warnings = [];
     errors = [];
+    processedRecords = new Map();
     static examples = [
         `<%= config.bin %> <%= command.id %>`,
         `<%= config.bin %> <%= command.id %> --dry-run`,
@@ -66,7 +69,11 @@ class Push extends core_1.Command {
         const { flags } = await this.parse(Push);
         const spinner = (0, ora_classic_1.default)();
         let sqlLogger = null;
+        const fileBackupManager = new file_backup_manager_1.FileBackupManager();
+        let hasActiveTransaction = false;
         const startTime = Date.now();
+        // Reset the processed records tracking for this push operation
+        this.processedRecords.clear();
         try {
             // Load configurations
             spinner.start('Loading configuration');
@@ -130,7 +137,7 @@ class Push extends core_1.Command {
                 }
             }
             // Find entity directories to process
-            const entityDirs = (0, provider_utils_1.findEntityDirectories)(config_manager_1.configManager.getOriginalCwd(), flags.dir, syncConfig?.directoryOrder);
+            const entityDirs = (0, provider_utils_1.findEntityDirectories)(config_manager_1.configManager.getOriginalCwd(), flags.dir, syncConfig?.directoryOrder, syncConfig?.ignoreDirectories);
             if (entityDirs.length === 0) {
                 this.error('No entity directories found');
             }
@@ -161,7 +168,9 @@ class Push extends core_1.Command {
                             default: false
                         });
                         if (!shouldContinue) {
-                            this.error('Push cancelled due to validation errors.');
+                            this.log(chalk_1.default.yellow('\n⚠️  Push cancelled due to validation errors.'));
+                            // Exit cleanly without throwing an error
+                            return;
                         }
                     }
                 }
@@ -169,31 +178,69 @@ class Push extends core_1.Command {
                     this.log(chalk_1.default.green('✓ Validation passed'));
                 }
             }
+            // Initialize file backup manager (unless in dry-run mode)
+            if (!flags['dry-run']) {
+                await fileBackupManager.initialize();
+                if (flags.verbose) {
+                    this.log('📁 File backup manager initialized');
+                }
+            }
             // Start a database transaction for the entire push operation (unless in dry-run mode)
             // IMPORTANT: We start the transaction AFTER metadata loading and validation to avoid
             // transaction conflicts with background refresh operations
             if (!flags['dry-run']) {
                 const { getDataProvider } = await Promise.resolve().then(() => __importStar(require('../../lib/provider-utils')));
                 const dataProvider = getDataProvider();
+                // Ensure we have SQLServerDataProvider for transaction support
+                if (!(dataProvider instanceof sqlserver_dataprovider_1.SQLServerDataProvider)) {
+                    const errorMsg = 'MetadataSync requires SQLServerDataProvider for transaction support. Current provider does not support transactions.';
+                    // Rollback file backups since we're not proceeding
+                    try {
+                        await fileBackupManager.rollback();
+                    }
+                    catch (rollbackError) {
+                        this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                    }
+                    this.error(errorMsg);
+                }
                 if (dataProvider && typeof dataProvider.BeginTransaction === 'function') {
                     try {
                         await dataProvider.BeginTransaction();
+                        hasActiveTransaction = true;
                         if (flags.verbose) {
                             this.log('🔄 Transaction started - all changes will be committed or rolled back as a unit');
                         }
                     }
                     catch (error) {
-                        this.warn('Failed to start transaction - changes will be committed individually');
-                        this.warn(`Transaction error: ${error instanceof Error ? error.message : String(error)}`);
+                        // Transaction start failure is critical - we should not proceed without it
+                        const errorMsg = `Failed to start database transaction: ${error instanceof Error ? error.message : String(error)}`;
+                        // Rollback file backups since we're not proceeding
+                        try {
+                            await fileBackupManager.rollback();
+                        }
+                        catch (rollbackError) {
+                            this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                        }
+                        this.error(errorMsg);
                     }
                 }
                 else {
-                    this.warn('Transaction support not available - changes will be committed individually');
+                    // No transaction support is also critical for data integrity
+                    const errorMsg = 'Transaction support not available - cannot ensure data integrity';
+                    // Rollback file backups since we're not proceeding
+                    try {
+                        await fileBackupManager.rollback();
+                    }
+                    catch (rollbackError) {
+                        this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                    }
+                    this.error(errorMsg);
                 }
             }
             // Process each entity directory
             let totalCreated = 0;
             let totalUpdated = 0;
+            let totalUnchanged = 0;
             let totalErrors = 0;
             for (const entityDir of entityDirs) {
                 const entityConfig = await (0, config_1.loadEntityConfig)(entityDir);
@@ -204,9 +251,34 @@ class Push extends core_1.Command {
                 if (flags.verbose) {
                     this.log(`\nProcessing ${entityConfig.entity} in ${entityDir}`);
                 }
-                const result = await this.processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig);
+                // Combine root ignoreDirectories with entity-level ignoreDirectories
+                const initialIgnoreDirectories = [
+                    ...(syncConfig?.ignoreDirectories || []),
+                    ...(entityConfig?.ignoreDirectories || [])
+                ];
+                const result = await this.processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig, fileBackupManager, initialIgnoreDirectories);
+                // Show per-directory summary
+                const dirName = path_1.default.relative(process.cwd(), entityDir) || '.';
+                const dirTotal = result.created + result.updated + result.unchanged;
+                if (dirTotal > 0 || result.errors > 0) {
+                    this.log(`\n📁 ${dirName}:`);
+                    this.log(`   Total processed: ${dirTotal} unique records`);
+                    if (result.created > 0) {
+                        this.log(`   ✓ Created: ${result.created}`);
+                    }
+                    if (result.updated > 0) {
+                        this.log(`   ✓ Updated: ${result.updated}`);
+                    }
+                    if (result.unchanged > 0) {
+                        this.log(`   - Unchanged: ${result.unchanged}`);
+                    }
+                    if (result.errors > 0) {
+                        this.log(`   ✗ Errors: ${result.errors}`);
+                    }
+                }
                 totalCreated += result.created;
                 totalUpdated += result.updated;
+                totalUnchanged += result.unchanged;
                 totalErrors += result.errors;
             }
             // Summary using FormattingService
@@ -216,15 +288,16 @@ class Push extends core_1.Command {
             this.log('\n' + formatter.formatSyncSummary('push', {
                 created: totalCreated,
                 updated: totalUpdated,
+                unchanged: totalUnchanged,
                 deleted: 0,
                 skipped: 0,
                 errors: totalErrors,
                 duration: endTime - startTime
             }));
             // Handle transaction commit/rollback
-            if (!flags['dry-run']) {
+            if (!flags['dry-run'] && hasActiveTransaction) {
                 const dataProvider = core_2.Metadata.Provider;
-                // Check if we have an active transaction
+                // We know we have an active transaction at this point
                 if (dataProvider) {
                     let shouldCommit = true;
                     // If there are any errors, always rollback
@@ -261,12 +334,18 @@ class Push extends core_1.Command {
                         if (shouldCommit) {
                             await dataProvider.CommitTransaction();
                             this.log('\n✅ All changes committed successfully');
+                            // Clean up file backups after successful commit
+                            await fileBackupManager.cleanup();
                         }
                         else {
                             // User chose to rollback or errors/warnings in CI mode
                             this.log('\n🔙 Rolling back all changes...');
+                            // Rollback database transaction
                             await dataProvider.RollbackTransaction();
-                            this.log('✅ Rollback completed - no changes were made to the database');
+                            // Rollback file changes
+                            this.log('🔙 Rolling back file changes...');
+                            await fileBackupManager.rollback();
+                            this.log('✅ Rollback completed - no changes were made to the database or files');
                         }
                     }
                     catch (error) {
@@ -274,10 +353,19 @@ class Push extends core_1.Command {
                         this.log('\n❌ Transaction error - attempting to roll back changes');
                         try {
                             await dataProvider.RollbackTransaction();
-                            this.log('✅ Rollback completed');
+                            this.log('✅ Database rollback completed');
                         }
                         catch (rollbackError) {
-                            this.log('❌ Rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
+                            this.log('❌ Database rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
+                        }
+                        // Also rollback file changes
+                        try {
+                            this.log('🔙 Rolling back file changes...');
+                            await fileBackupManager.rollback();
+                            this.log('✅ File rollback completed');
+                        }
+                        catch (fileRollbackError) {
+                            this.log('❌ File rollback failed: ' + (fileRollbackError instanceof Error ? fileRollbackError.message : String(fileRollbackError)));
                         }
                         throw error;
                     }
@@ -290,20 +378,30 @@ class Push extends core_1.Command {
         }
         catch (error) {
             spinner.fail('Push failed');
-            // Try to rollback the transaction if one is active
+            // Try to rollback the transaction and files if not in dry-run mode
             if (!flags['dry-run']) {
                 const { getDataProvider } = await Promise.resolve().then(() => __importStar(require('../../lib/provider-utils')));
                 const dataProvider = getDataProvider();
-                if (dataProvider && typeof dataProvider.RollbackTransaction === 'function') {
+                // Rollback database transaction if we have one
+                if (hasActiveTransaction && dataProvider && typeof dataProvider.RollbackTransaction === 'function') {
                     try {
-                        this.log('\n🔙 Rolling back transaction due to error...');
+                        this.log('\n🔙 Rolling back database transaction due to error...');
                         await dataProvider.RollbackTransaction();
-                        this.log('✅ Rollback completed - no changes were made to the database');
+                        this.log('✅ Database rollback completed');
                     }
                     catch (rollbackError) {
-                        this.log('❌ Rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
+                        this.log('❌ Database rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
                     }
                 }
+                // Rollback file changes
+                try {
+                    this.log('🔙 Rolling back file changes...');
+                    await fileBackupManager.rollback();
+                    this.log('✅ File rollback completed - all files restored to original state');
+                }
+                catch (fileRollbackError) {
+                    this.log('❌ File rollback failed: ' + (fileRollbackError instanceof Error ? fileRollbackError.message : String(fileRollbackError)));
+                }
             }
             // Enhanced error logging for debugging
             this.log('\n=== Push Error Details ===');
@@ -353,8 +451,8 @@ class Push extends core_1.Command {
             process.exit(0);
         }
     }
-    async processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig) {
-        const result = { created: 0, updated: 0, errors: 0 };
+    async processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig, fileBackupManager, parentIgnoreDirectories) {
+        const result = { created: 0, updated: 0, unchanged: 0, errors: 0 };
         // Find files matching the configured pattern
         const pattern = entityConfig.filePattern || '*.json';
         const jsonFiles = await (0, fast_glob_1.default)(pattern, {
@@ -363,15 +461,83 @@ class Push extends core_1.Command {
             dot: true, // Include dotfiles (files starting with .)
             onlyFiles: true
         });
+        // Check if no JSON files were found
+        if (jsonFiles.length === 0) {
+            const relativePath = path_1.default.relative(process.cwd(), entityDir) || '.';
+            const parentPath = path_1.default.dirname(entityDir);
+            const dirName = path_1.default.basename(entityDir);
+            // Check if this is a subdirectory (not a top-level entity directory)
+            const isSubdirectory = parentPath !== path_1.default.resolve(config_manager_1.configManager.getOriginalCwd(), flags.dir || '.');
+            if (isSubdirectory) {
+                // For subdirectories, make it a warning instead of an error
+                let warningMessage = `No JSON files found in ${relativePath} matching pattern: ${pattern}`;
+                // Try to be more helpful by checking what files do exist
+                const allFiles = await (0, fast_glob_1.default)('*', {
+                    cwd: entityDir,
+                    onlyFiles: true,
+                    dot: true
+                });
+                if (allFiles.length > 0) {
+                    warningMessage += `\n  Files found: ${allFiles.slice(0, 3).join(', ')}`;
+                    if (allFiles.length > 3) {
+                        warningMessage += ` (and ${allFiles.length - 3} more)`;
+                    }
+                }
+                const rootConfigPath = path_1.default.join(config_manager_1.configManager.getOriginalCwd(), flags.dir || '.', '.mj-sync.json');
+                warningMessage += `\n  💡 If this directory should be ignored, add "${dirName}" to the "ignoreDirectories" array in:\n     ${rootConfigPath}`;
+                this.warn(warningMessage);
+                return result; // Return early without processing further
+            }
+            else {
+                // For top-level entity directories, this is still an error
+                const configFile = path_1.default.join(entityDir, '.mj-sync.json');
+                let errorMessage = `No JSON files found in ${relativePath} matching pattern: ${pattern}\n`;
+                errorMessage += `\nPlease check:\n`;
+                errorMessage += `  1. Files exist with the expected extension (.json)\n`;
+                errorMessage += `  2. The filePattern in ${configFile} matches your files\n`;
+                errorMessage += `  3. Files are not in ignored patterns: .mj-sync.json, .mj-folder.json, *.backup\n`;
+                // Try to be more helpful by checking what files do exist
+                const allFiles = await (0, fast_glob_1.default)('*', {
+                    cwd: entityDir,
+                    onlyFiles: true,
+                    dot: true
+                });
+                if (allFiles.length > 0) {
+                    errorMessage += `\nFiles found in directory: ${allFiles.slice(0, 5).join(', ')}`;
+                    if (allFiles.length > 5) {
+                        errorMessage += ` (and ${allFiles.length - 5} more)`;
+                    }
+                }
+                throw new Error(errorMessage);
+            }
+        }
         if (flags.verbose) {
             this.log(`Processing ${jsonFiles.length} records in ${path_1.default.relative(process.cwd(), entityDir) || '.'}`);
         }
         // First, process all JSON files in this directory
-        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result);
+        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager);
         // Then, recursively process subdirectories
         const entries = await fs_extra_1.default.readdir(entityDir, { withFileTypes: true });
         for (const entry of entries) {
             if (entry.isDirectory() && !entry.name.startsWith('.')) {
+                // Build cumulative ignore list: parent + current directory's ignores
+                const currentDirConfig = await (0, config_1.loadSyncConfig)(entityDir);
+                const currentEntityConfig = await (0, config_1.loadEntityConfig)(entityDir);
+                const cumulativeIgnoreDirectories = [
+                    ...(parentIgnoreDirectories || []),
+                    ...(currentDirConfig?.ignoreDirectories || []),
+                    ...(currentEntityConfig?.ignoreDirectories || [])
+                ];
+                // Check if this directory should be ignored
+                if (cumulativeIgnoreDirectories.some((pattern) => {
+                    // Simple pattern matching: exact name or ends with pattern
+                    return entry.name === pattern || entry.name.endsWith(pattern);
+                })) {
+                    if (flags.verbose) {
+                        this.log(`  Ignoring directory: ${entry.name} (matched ignore pattern)`);
+                    }
+                    continue;
+                }
                 const subDir = path_1.default.join(entityDir, entry.name);
                 // Load subdirectory config and merge with parent config
                 let subEntityConfig = { ...entityConfig };
@@ -392,48 +558,69 @@ class Push extends core_1.Command {
                         }
                     };
                 }
-                // Process subdirectory with merged config
-                const subResult = await this.processEntityDirectory(subDir, subEntityConfig, syncEngine, flags, syncConfig);
+                // Process subdirectory with merged config and cumulative ignore directories
+                const subResult = await this.processEntityDirectory(subDir, subEntityConfig, syncEngine, flags, syncConfig, fileBackupManager, cumulativeIgnoreDirectories);
                 result.created += subResult.created;
                 result.updated += subResult.updated;
+                result.unchanged += subResult.unchanged;
                 result.errors += subResult.errors;
             }
         }
         return result;
     }
-    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result) {
+    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager) {
         if (jsonFiles.length === 0) {
             return;
         }
         const spinner = (0, ora_classic_1.default)();
         spinner.start('Processing records');
-        let totalRecords = 0;
         for (const file of jsonFiles) {
             try {
                 const filePath = path_1.default.join(entityDir, file);
-                const fileContent = await fs_extra_1.default.readJson(filePath);
+                // Backup the file before any modifications (unless dry-run)
+                if (!flags['dry-run'] && fileBackupManager) {
+                    await fileBackupManager.backupFile(filePath);
+                }
+                // Parse JSON with line number tracking
+                const { content: fileContent, lineNumbers } = await this.parseJsonWithLineNumbers(filePath);
                 // Process templates in the loaded content
                 const processedContent = await syncEngine.processTemplates(fileContent, entityDir);
                 // Check if the file contains a single record or an array of records
                 const isArray = Array.isArray(processedContent);
                 const records = isArray ? processedContent : [processedContent];
-                totalRecords += records.length;
                 // Build and process defaults (including lookups)
                 const defaults = await syncEngine.buildDefaults(filePath, entityConfig);
                 // Process each record in the file
                 for (let i = 0; i < records.length; i++) {
                     const recordData = records[i];
                     // Process the record
-                    const isNew = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined);
+                    const recordLineNumber = lineNumbers.get(i); // Get line number for this array index
+                    const pushResult = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined, fileBackupManager, recordLineNumber);
                     if (!flags['dry-run']) {
-                        if (isNew) {
-                            result.created++;
+                        // Don't count duplicates in stats
+                        if (!pushResult.isDuplicate) {
+                            if (pushResult.isNew) {
+                                result.created++;
+                            }
+                            else if (pushResult.wasActuallyUpdated) {
+                                result.updated++;
+                            }
+                            else {
+                                result.unchanged++;
+                            }
                         }
-                        else {
-                            result.updated++;
+                        // Add related entity stats
+                        if (pushResult.relatedStats) {
+                            result.created += pushResult.relatedStats.created;
+                            result.updated += pushResult.relatedStats.updated;
+                            result.unchanged += pushResult.relatedStats.unchanged;
+                            // Debug logging for related entities
+                            if (flags.verbose && pushResult.relatedStats.unchanged > 0) {
+                                this.log(`   Related entities: ${pushResult.relatedStats.unchanged} unchanged`);
+                            }
                         }
                     }
-                    spinner.text = `Processing records (${result.created + result.updated + result.errors}/${totalRecords})`;
+                    spinner.text = `Processing records (${result.created + result.updated + result.unchanged + result.errors} processed)`;
                 }
                 // Write back the entire file if it's an array
                 if (isArray && !flags['dry-run']) {
@@ -446,21 +633,43 @@ class Push extends core_1.Command {
                 const fullErrorMessage = `Failed to process ${file}: ${errorMessage}`;
                 this.errors.push(fullErrorMessage);
                 this.error(fullErrorMessage, { exit: false });
+                this.log('   ⚠️  This error will cause all changes to be rolled back at the end of processing');
             }
         }
         if (flags.verbose) {
-            spinner.succeed(`Processed ${totalRecords} records from ${jsonFiles.length} files`);
+            spinner.succeed(`Processed ${result.created + result.updated + result.unchanged} records from ${jsonFiles.length} files`);
         }
         else {
             spinner.stop();
         }
     }
-    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex) {
+    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex, fileBackupManager, lineNumber) {
         // Load or create entity
         let entity = null;
         let isNew = false;
         if (recordData.primaryKey) {
             entity = await syncEngine.loadEntity(entityName, recordData.primaryKey);
+            // Warn if record has primaryKey but wasn't found
+            if (!entity) {
+                const pkDisplay = Object.entries(recordData.primaryKey)
+                    .map(([key, value]) => `${key}=${value}`)
+                    .join(', ');
+                // Load sync config to check autoCreateMissingRecords setting
+                const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
+                if (!autoCreate) {
+                    const fileRef = lineNumber ? `${fileName}:${lineNumber}` : fileName;
+                    this.warn(`⚠️  Record not found: ${entityName} with primaryKey {${pkDisplay}} at ${fileRef}`);
+                    this.warn(`   To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`);
+                    // Skip this record
+                    return { isNew: false, wasActuallyUpdated: false, isDuplicate: false };
+                }
+                else {
+                    if (verbose) {
+                        this.log(`   Auto-creating missing ${entityName} record with primaryKey {${pkDisplay}}`);
+                    }
+                }
+            }
         }
         if (!entity) {
             // New record
@@ -507,7 +716,7 @@ class Push extends core_1.Command {
                 try {
                     const processedValue = await syncEngine.processFieldValue(value, baseDir, null, null);
                     if (verbose) {
-                        this.log(`  Setting ${field}: ${JSON.stringify(value)} -> ${JSON.stringify(processedValue)}`);
+                        this.log(`  Setting ${field}: ${this.formatFieldValue(value)} -> ${this.formatFieldValue(processedValue)}`);
                     }
                     entity[field] = processedValue;
                 }
@@ -521,7 +730,46 @@ class Push extends core_1.Command {
         }
         if (dryRun) {
             this.log(`Would ${isNew ? 'create' : 'update'} ${entityName} record`);
-            return isNew;
+            return { isNew, wasActuallyUpdated: true, isDuplicate: false, relatedStats: undefined };
+        }
+        // Check for duplicate processing (but only for existing records that were loaded)
+        let isDuplicate = false;
+        if (!isNew && entity) {
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            isDuplicate = this.checkAndTrackRecord(entityName, entity, fullFilePath, arrayIndex, lineNumber);
+        }
+        // Check if the record is dirty before saving
+        let wasActuallyUpdated = false;
+        if (!isNew && entity.Dirty) {
+            // Record is dirty, get the changes
+            const changes = entity.GetChangesSinceLastSave();
+            const changeKeys = Object.keys(changes);
+            if (changeKeys.length > 0) {
+                wasActuallyUpdated = true;
+                // Get primary key info for display
+                const entityInfo = syncEngine.getEntityInfo(entityName);
+                const primaryKeyDisplay = [];
+                if (entityInfo) {
+                    for (const pk of entityInfo.PrimaryKeys) {
+                        primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                    }
+                }
+                this.log(''); // Add newline before update output
+                this.log(`📝 Updating ${entityName} record:`);
+                if (primaryKeyDisplay.length > 0) {
+                    this.log(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+                }
+                this.log(`   Changes:`);
+                for (const fieldName of changeKeys) {
+                    const field = entity.GetFieldByName(fieldName);
+                    const oldValue = field ? field.OldValue : undefined;
+                    const newValue = changes[fieldName];
+                    this.log(`     ${fieldName}: ${this.formatFieldValue(oldValue)} → ${this.formatFieldValue(newValue)}`);
+                }
+            }
+        }
+        else if (isNew) {
+            wasActuallyUpdated = true;
         }
         // Save the record
         const saved = await entity.Save();
@@ -534,9 +782,12 @@ class Push extends core_1.Command {
             throw new Error(`Failed to save record: ${errors}`);
         }
         // Process related entities after saving parent
+        let relatedStats;
         if (recordData.relatedEntities && !dryRun) {
-            await this.processRelatedEntities(recordData.relatedEntities, entity, entity, // root is same as parent for top level
-            baseDir, syncEngine, verbose);
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            relatedStats = await this.processRelatedEntities(recordData.relatedEntities, entity, entity, // root is same as parent for top level
+            baseDir, syncEngine, verbose, fileBackupManager, 1, // indentLevel
+            fullFilePath, arrayIndex);
         }
         // Update the local file with new primary key if created
         if (isNew) {
@@ -548,6 +799,9 @@ class Push extends core_1.Command {
                 }
                 recordData.primaryKey = newPrimaryKey;
             }
+            // Track the new record now that we have its primary key
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            this.checkAndTrackRecord(entityName, entity, fullFilePath, arrayIndex, lineNumber);
         }
         // Always update sync metadata
         // This ensures related entities are persisted with their metadata
@@ -561,10 +815,11 @@ class Push extends core_1.Command {
             const filePath = path_1.default.join(baseDir, fileName);
             await fs_extra_1.default.writeJson(filePath, recordData, { spaces: 2 });
         }
-        return isNew;
+        return { isNew, wasActuallyUpdated, isDuplicate, relatedStats };
     }
-    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, indentLevel = 1) {
+    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, fileBackupManager, indentLevel = 1, parentFilePath, parentArrayIndex) {
         const indent = '  '.repeat(indentLevel);
+        const stats = { created: 0, updated: 0, unchanged: 0 };
         for (const [entityName, records] of Object.entries(relatedEntities)) {
             if (verbose) {
                 this.log(`${indent}↳ Processing ${records.length} related ${entityName} records`);
@@ -576,6 +831,27 @@ class Push extends core_1.Command {
                     let isNew = false;
                     if (relatedRecord.primaryKey) {
                         entity = await syncEngine.loadEntity(entityName, relatedRecord.primaryKey);
+                        // Warn if record has primaryKey but wasn't found
+                        if (!entity) {
+                            const pkDisplay = Object.entries(relatedRecord.primaryKey)
+                                .map(([key, value]) => `${key}=${value}`)
+                                .join(', ');
+                            // Load sync config to check autoCreateMissingRecords setting
+                            const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                            const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
+                            if (!autoCreate) {
+                                const fileRef = parentFilePath ? path_1.default.relative(config_manager_1.configManager.getOriginalCwd(), parentFilePath) : 'unknown';
+                                this.warn(`${indent}⚠️  Related record not found: ${entityName} with primaryKey {${pkDisplay}} at ${fileRef}`);
+                                this.warn(`${indent}   To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`);
+                                // Skip this record
+                                continue;
+                            }
+                            else {
+                                if (verbose) {
+                                    this.log(`${indent}   Auto-creating missing related ${entityName} record with primaryKey {${pkDisplay}}`);
+                                }
+                            }
+                        }
                     }
                     if (!entity) {
                         entity = await syncEngine.createEntityObject(entityName);
@@ -614,7 +890,7 @@ class Push extends core_1.Command {
                             try {
                                 const processedValue = await syncEngine.processFieldValue(value, baseDir, parentEntity, rootEntity);
                                 if (verbose) {
-                                    this.log(`${indent}  Setting ${field}: ${JSON.stringify(value)} -> ${JSON.stringify(processedValue)}`);
+                                    this.log(`${indent}  Setting ${field}: ${this.formatFieldValue(value)} -> ${this.formatFieldValue(processedValue)}`);
                                 }
                                 entity[field] = processedValue;
                             }
@@ -626,6 +902,46 @@ class Push extends core_1.Command {
                             this.warn(`${indent}  Field '${field}' does not exist on entity '${entityName}'`);
                         }
                     }
+                    // Check for duplicate processing (but only for existing records that were loaded)
+                    let isDuplicate = false;
+                    if (!isNew && entity) {
+                        // Use parent file path for related entities since they're defined in the parent's file
+                        const relatedFilePath = parentFilePath || path_1.default.join(baseDir, 'unknown');
+                        isDuplicate = this.checkAndTrackRecord(entityName, entity, relatedFilePath, parentArrayIndex);
+                    }
+                    // Check if the record is dirty before saving
+                    let wasActuallyUpdated = false;
+                    if (!isNew && entity.Dirty) {
+                        // Record is dirty, get the changes
+                        const changes = entity.GetChangesSinceLastSave();
+                        const changeKeys = Object.keys(changes);
+                        if (changeKeys.length > 0) {
+                            wasActuallyUpdated = true;
+                            // Get primary key info for display
+                            const entityInfo = syncEngine.getEntityInfo(entityName);
+                            const primaryKeyDisplay = [];
+                            if (entityInfo) {
+                                for (const pk of entityInfo.PrimaryKeys) {
+                                    primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                                }
+                            }
+                            this.log(''); // Add newline before update output
+                            this.log(`${indent}📝 Updating related ${entityName} record:`);
+                            if (primaryKeyDisplay.length > 0) {
+                                this.log(`${indent}   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+                            }
+                            this.log(`${indent}   Changes:`);
+                            for (const fieldName of changeKeys) {
+                                const field = entity.GetFieldByName(fieldName);
+                                const oldValue = field ? field.OldValue : undefined;
+                                const newValue = changes[fieldName];
+                                this.log(`${indent}     ${fieldName}: ${this.formatFieldValue(oldValue)} → ${this.formatFieldValue(newValue)}`);
+                            }
+                        }
+                    }
+                    else if (isNew) {
+                        wasActuallyUpdated = true;
+                    }
                     // Save the related entity
                     const saved = await entity.Save();
                     if (!saved) {
@@ -636,9 +952,24 @@ class Push extends core_1.Command {
                         const errors = entity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || 'Unknown error';
                         throw new Error(`Failed to save related ${entityName}: ${errors}`);
                     }
-                    if (verbose) {
+                    // Update stats - don't count duplicates
+                    if (!isDuplicate) {
+                        if (isNew) {
+                            stats.created++;
+                        }
+                        else if (wasActuallyUpdated) {
+                            stats.updated++;
+                        }
+                        else {
+                            stats.unchanged++;
+                        }
+                    }
+                    if (verbose && wasActuallyUpdated) {
                         this.log(`${indent}  ✓ ${isNew ? 'Created' : 'Updated'} ${entityName} record`);
                     }
+                    else if (verbose && !wasActuallyUpdated) {
+                        this.log(`${indent}  - No changes to ${entityName} record`);
+                    }
                     // Update the related record with primary key and sync metadata
                     const entityInfo = syncEngine.getEntityInfo(entityName);
                     if (entityInfo) {
@@ -648,6 +979,9 @@ class Push extends core_1.Command {
                             for (const pk of entityInfo.PrimaryKeys) {
                                 relatedRecord.primaryKey[pk.Name] = entity.Get(pk.Name);
                             }
+                            // Track the new related entity now that we have its primary key
+                            const relatedFilePath = parentFilePath || path_1.default.join(baseDir, 'unknown');
+                            this.checkAndTrackRecord(entityName, entity, relatedFilePath, parentArrayIndex);
                         }
                         // Always update sync metadata
                         relatedRecord.sync = {
@@ -657,7 +991,11 @@ class Push extends core_1.Command {
                     }
                     // Process nested related entities if any
                     if (relatedRecord.relatedEntities) {
-                        await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, indentLevel + 1);
+                        const nestedStats = await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, fileBackupManager, indentLevel + 1, parentFilePath, parentArrayIndex);
+                        // Accumulate nested stats
+                        stats.created += nestedStats.created;
+                        stats.updated += nestedStats.updated;
+                        stats.unchanged += nestedStats.unchanged;
                     }
                 }
                 catch (error) {
@@ -665,6 +1003,128 @@ class Push extends core_1.Command {
                 }
             }
         }
+        return stats;
+    }
+    /**
+     * Generate a unique tracking key for a record based on entity name and primary key values
+     */
+    generateRecordKey(entityName, entity) {
+        const entityInfo = entity.EntityInfo;
+        const primaryKeyValues = [];
+        if (entityInfo && entityInfo.PrimaryKeys.length > 0) {
+            for (const pk of entityInfo.PrimaryKeys) {
+                const value = entity.Get(pk.Name);
+                primaryKeyValues.push(`${pk.Name}:${value}`);
+            }
+        }
+        return `${entityName}|${primaryKeyValues.join('|')}`;
+    }
+    /**
+     * Check if a record has already been processed and warn if duplicate
+     */
+    checkAndTrackRecord(entityName, entity, filePath, arrayIndex, lineNumber) {
+        const recordKey = this.generateRecordKey(entityName, entity);
+        const existing = this.processedRecords.get(recordKey);
+        if (existing) {
+            const primaryKeyDisplay = entity.EntityInfo?.PrimaryKeys
+                .map(pk => `${pk.Name}: ${entity.Get(pk.Name)}`)
+                .join(', ') || 'unknown';
+            // Format file location with clickable link for VSCode
+            // Create maps with just the line numbers we have
+            const currentLineMap = lineNumber ? new Map([[arrayIndex || 0, lineNumber]]) : undefined;
+            const originalLineMap = existing.lineNumber ? new Map([[existing.arrayIndex || 0, existing.lineNumber]]) : undefined;
+            const currentLocation = this.formatFileLocation(filePath, arrayIndex, currentLineMap);
+            const originalLocation = this.formatFileLocation(existing.filePath, existing.arrayIndex, originalLineMap);
+            this.warn(`⚠️  Duplicate record detected for ${entityName} (${primaryKeyDisplay})`);
+            this.warn(`   Current location:  ${currentLocation}`);
+            this.warn(`   Original location: ${originalLocation}`);
+            this.warn(`   The duplicate update will proceed, but you should review your data for unintended duplicates.`);
+            return true; // is duplicate
+        }
+        // Track the record with its source location
+        this.processedRecords.set(recordKey, {
+            filePath: filePath || 'unknown',
+            arrayIndex,
+            lineNumber
+        });
+        return false; // not duplicate
+    }
+    /**
+     * Format field value for console display
+     */
+    formatFieldValue(value, maxLength = 50) {
+        // Convert value to string representation
+        let strValue = JSON.stringify(value);
+        // Trim the string
+        strValue = strValue.trim();
+        // If it's longer than maxLength, truncate and add ellipsis
+        if (strValue.length > maxLength) {
+            return strValue.substring(0, maxLength) + '...';
+        }
+        return strValue;
+    }
+    /**
+     * Parse JSON file and track line numbers for array elements
+     */
+    async parseJsonWithLineNumbers(filePath) {
+        const fileText = await fs_extra_1.default.readFile(filePath, 'utf-8');
+        const lines = fileText.split('\n');
+        const lineNumbers = new Map();
+        // Parse the JSON
+        const content = JSON.parse(fileText);
+        // If it's an array, try to find where each element starts
+        if (Array.isArray(content)) {
+            let inString = false;
+            let bracketDepth = 0;
+            let currentIndex = -1;
+            for (let lineNum = 0; lineNum < lines.length; lineNum++) {
+                const line = lines[lineNum];
+                // Simple tracking of string boundaries and bracket depth
+                for (let i = 0; i < line.length; i++) {
+                    const char = line[i];
+                    const prevChar = i > 0 ? line[i - 1] : '';
+                    if (char === '"' && prevChar !== '\\') {
+                        inString = !inString;
+                    }
+                    if (!inString) {
+                        if (char === '{') {
+                            bracketDepth++;
+                            // If we're at depth 1 in the main array, this is a new object
+                            if (bracketDepth === 1 && line.trim().startsWith('{')) {
+                                currentIndex++;
+                                lineNumbers.set(currentIndex, lineNum + 1); // 1-based line numbers
+                            }
+                        }
+                        else if (char === '}') {
+                            bracketDepth--;
+                        }
+                    }
+                }
+            }
+        }
+        return { content, lineNumbers };
+    }
+    /**
+     * Format file location with clickable link for VSCode
+     */
+    formatFileLocation(filePath, arrayIndex, lineNumbers) {
+        if (!filePath || filePath === 'unknown') {
+            return 'unknown';
+        }
+        // Get absolute path for better VSCode integration
+        const absolutePath = path_1.default.resolve(filePath);
+        // Try to get actual line number from our tracking
+        let lineNumber = 1;
+        if (arrayIndex !== undefined && lineNumbers && lineNumbers.has(arrayIndex)) {
+            lineNumber = lineNumbers.get(arrayIndex);
+        }
+        else if (arrayIndex !== undefined) {
+            // Fallback estimation if we don't have actual line numbers
+            lineNumber = 2 + (arrayIndex * 15);
+        }
+        // Create clickable file path for VSCode - format: file:line
+        // VSCode will make this clickable in the terminal
+        return `${absolutePath}:${lineNumber}`;
     }
 }
 exports.default = Push;