@memberjunction/metadata-sync 2.50.0 → 2.52.0

package/dist/commands/push/index.js

@@ -29,16 +29,23 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const core_1 = require("@oclif/core");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
+const prompts_1 = require("@inquirer/prompts");
 const ora_classic_1 = __importDefault(require("ora-classic"));
 const fast_glob_1 = __importDefault(require("fast-glob"));
+const chalk_1 = __importDefault(require("chalk"));
 const config_1 = require("../../config");
 const provider_utils_1 = require("../../lib/provider-utils");
-const provider_utils_2 = require("../../lib/provider-utils");
+const core_2 = require("@memberjunction/core");
 const config_manager_1 = require("../../lib/config-manager");
 const singleton_manager_1 = require("../../lib/singleton-manager");
+const sqlserver_dataprovider_1 = require("@memberjunction/sqlserver-dataprovider");
 const global_1 = require("@memberjunction/global");
+const file_backup_manager_1 = require("../../lib/file-backup-manager");
 class Push extends core_1.Command {
     static description = 'Push local file changes to the database';
+    warnings = [];
+    errors = [];
+    processedRecords = new Map();
     static examples = [
         `<%= config.bin %> <%= command.id %>`,
         `<%= config.bin %> <%= command.id %> --dry-run`,
@@ -50,11 +57,23 @@ class Push extends core_1.Command {
         'dry-run': core_1.Flags.boolean({ description: 'Show what would be pushed without actually pushing' }),
         ci: core_1.Flags.boolean({ description: 'CI mode - no prompts, fail on issues' }),
         verbose: core_1.Flags.boolean({ char: 'v', description: 'Show detailed field-level output' }),
+        'no-validate': core_1.Flags.boolean({ description: 'Skip validation before push' }),
     };
+    // Override warn to collect warnings
+    warn(input) {
+        const message = typeof input === 'string' ? input : input.message;
+        this.warnings.push(message);
+        return super.warn(input);
+    }
     async run() {
         const { flags } = await this.parse(Push);
         const spinner = (0, ora_classic_1.default)();
         let sqlLogger = null;
+        const fileBackupManager = new file_backup_manager_1.FileBackupManager();
+        let hasActiveTransaction = false;
+        const startTime = Date.now();
+        // Reset the processed records tracking for this push operation
+        this.processedRecords.clear();
         try {
             // Load configurations
             spinner.start('Loading configuration');
@@ -72,7 +91,12 @@ class Push extends core_1.Command {
             // Initialize sync engine using singleton pattern
             const syncEngine = await (0, singleton_manager_1.getSyncEngine)((0, provider_utils_1.getSystemUser)());
             // Show success after all initialization is complete
-            spinner.succeed('Configuration and metadata loaded');
+            if (flags.verbose) {
+                spinner.succeed('Configuration and metadata loaded');
+            }
+            else {
+                spinner.stop();
+            }
             // Initialize SQL logging AFTER provider setup is complete
             if (syncConfig?.sqlLogging?.enabled) {
                 const outputDir = syncConfig.sqlLogging.outputDirectory || './sql_logging';
@@ -96,18 +120,20 @@ class Push extends core_1.Command {
                 // Import and access the data provider from the provider utils
                 const { getDataProvider } = await Promise.resolve().then(() => __importStar(require('../../lib/provider-utils')));
                 const dataProvider = getDataProvider();
-                if (dataProvider && typeof dataProvider.createSqlLogger === 'function') {
-                    sqlLogger = await dataProvider.createSqlLogger(logFilePath, {
+                if (dataProvider && typeof dataProvider.CreateSqlLogger === 'function') {
+                    sqlLogger = await dataProvider.CreateSqlLogger(logFilePath, {
                         formatAsMigration,
                         description: 'MetadataSync Push Operation',
                         statementTypes: 'mutations', // Only log mutations (data changes)
                         batchSeparator: 'GO', // Add GO statements for SQL Server batch processing
                         prettyPrint: true // Enable pretty printing for readable output
                     });
-                    this.log(`📝 SQL logging enabled: ${path_1.default.relative(process.cwd(), logFilePath)}`);
+                    if (flags.verbose) {
+                        this.log(`📝 SQL logging enabled: ${path_1.default.relative(process.cwd(), logFilePath)}`);
+                    }
                 }
                 else {
-                    this.warn('SQL logging requested but data provider does not support createSqlLogger');
+                    this.warn('SQL logging requested but data provider does not support CreateSqlLogger');
                 }
             }
             // Find entity directories to process
@@ -115,10 +141,104 @@ class Push extends core_1.Command {
             if (entityDirs.length === 0) {
                 this.error('No entity directories found');
             }
-            this.log(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
+            if (flags.verbose) {
+                this.log(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
+            }
+            // Run validation unless disabled
+            if (!flags['no-validate']) {
+                const { ValidationService } = await Promise.resolve().then(() => __importStar(require('../../services/ValidationService')));
+                const { FormattingService } = await Promise.resolve().then(() => __importStar(require('../../services/FormattingService')));
+                spinner.start('Validating metadata...');
+                const validator = new ValidationService({ verbose: flags.verbose });
+                const formatter = new FormattingService();
+                const targetDir = flags.dir ? path_1.default.resolve(config_manager_1.configManager.getOriginalCwd(), flags.dir) : config_manager_1.configManager.getOriginalCwd();
+                const validationResult = await validator.validateDirectory(targetDir);
+                spinner.stop();
+                if (!validationResult.isValid || validationResult.warnings.length > 0) {
+                    // Show validation results
+                    this.log('\n' + formatter.formatValidationResult(validationResult, flags.verbose));
+                    if (!validationResult.isValid) {
+                        // In CI mode, fail immediately
+                        if (flags.ci) {
+                            this.error('Validation failed. Cannot proceed with push.');
+                        }
+                        // Otherwise, ask for confirmation
+                        const shouldContinue = await (0, prompts_1.confirm)({
+                            message: 'Validation failed with errors. Do you want to continue anyway?',
+                            default: false
+                        });
+                        if (!shouldContinue) {
+                            this.error('Push cancelled due to validation errors.');
+                        }
+                    }
+                }
+                else {
+                    this.log(chalk_1.default.green('✓ Validation passed'));
+                }
+            }
+            // Initialize file backup manager (unless in dry-run mode)
+            if (!flags['dry-run']) {
+                await fileBackupManager.initialize();
+                if (flags.verbose) {
+                    this.log('📁 File backup manager initialized');
+                }
+            }
+            // Start a database transaction for the entire push operation (unless in dry-run mode)
+            // IMPORTANT: We start the transaction AFTER metadata loading and validation to avoid
+            // transaction conflicts with background refresh operations
+            if (!flags['dry-run']) {
+                const { getDataProvider } = await Promise.resolve().then(() => __importStar(require('../../lib/provider-utils')));
+                const dataProvider = getDataProvider();
+                // Ensure we have SQLServerDataProvider for transaction support
+                if (!(dataProvider instanceof sqlserver_dataprovider_1.SQLServerDataProvider)) {
+                    const errorMsg = 'MetadataSync requires SQLServerDataProvider for transaction support. Current provider does not support transactions.';
+                    // Rollback file backups since we're not proceeding
+                    try {
+                        await fileBackupManager.rollback();
+                    }
+                    catch (rollbackError) {
+                        this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                    }
+                    this.error(errorMsg);
+                }
+                if (dataProvider && typeof dataProvider.BeginTransaction === 'function') {
+                    try {
+                        await dataProvider.BeginTransaction();
+                        hasActiveTransaction = true;
+                        if (flags.verbose) {
+                            this.log('🔄 Transaction started - all changes will be committed or rolled back as a unit');
+                        }
+                    }
+                    catch (error) {
+                        // Transaction start failure is critical - we should not proceed without it
+                        const errorMsg = `Failed to start database transaction: ${error instanceof Error ? error.message : String(error)}`;
+                        // Rollback file backups since we're not proceeding
+                        try {
+                            await fileBackupManager.rollback();
+                        }
+                        catch (rollbackError) {
+                            this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                        }
+                        this.error(errorMsg);
+                    }
+                }
+                else {
+                    // No transaction support is also critical for data integrity
+                    const errorMsg = 'Transaction support not available - cannot ensure data integrity';
+                    // Rollback file backups since we're not proceeding
+                    try {
+                        await fileBackupManager.rollback();
+                    }
+                    catch (rollbackError) {
+                        this.warn(`Failed to rollback file backup initialization: ${rollbackError}`);
+                    }
+                    this.error(errorMsg);
+                }
+            }
             // Process each entity directory
             let totalCreated = 0;
             let totalUpdated = 0;
+            let totalUnchanged = 0;
             let totalErrors = 0;
             for (const entityDir of entityDirs) {
                 const entityConfig = await (0, config_1.loadEntityConfig)(entityDir);
@@ -126,23 +246,156 @@ class Push extends core_1.Command {
                     this.warn(`Skipping ${entityDir} - no valid entity configuration`);
                     continue;
                 }
-                this.log(`\nProcessing ${entityConfig.entity} in ${entityDir}`);
-                const result = await this.processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig);
+                if (flags.verbose) {
+                    this.log(`\nProcessing ${entityConfig.entity} in ${entityDir}`);
+                }
+                const result = await this.processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig, fileBackupManager);
+                // Show per-directory summary
+                const dirName = path_1.default.relative(process.cwd(), entityDir) || '.';
+                const dirTotal = result.created + result.updated + result.unchanged;
+                if (dirTotal > 0 || result.errors > 0) {
+                    this.log(`\n📁 ${dirName}:`);
+                    this.log(`   Total processed: ${dirTotal} unique records`);
+                    if (result.created > 0) {
+                        this.log(`   ✓ Created: ${result.created}`);
+                    }
+                    if (result.updated > 0) {
+                        this.log(`   ✓ Updated: ${result.updated}`);
+                    }
+                    if (result.unchanged > 0) {
+                        this.log(`   - Unchanged: ${result.unchanged}`);
+                    }
+                    if (result.errors > 0) {
+                        this.log(`   ✗ Errors: ${result.errors}`);
+                    }
+                }
                 totalCreated += result.created;
                 totalUpdated += result.updated;
+                totalUnchanged += result.unchanged;
                 totalErrors += result.errors;
             }
-            // Summary
-            this.log('\n=== Push Summary ===');
-            this.log(`Created: ${totalCreated}`);
-            this.log(`Updated: ${totalUpdated}`);
-            this.log(`Errors: ${totalErrors}`);
-            if (totalErrors > 0 && flags.ci) {
-                this.error('Push failed with errors in CI mode');
+            // Summary using FormattingService
+            const endTime = Date.now();
+            const { FormattingService } = await Promise.resolve().then(() => __importStar(require('../../services/FormattingService')));
+            const formatter = new FormattingService();
+            this.log('\n' + formatter.formatSyncSummary('push', {
+                created: totalCreated,
+                updated: totalUpdated,
+                unchanged: totalUnchanged,
+                deleted: 0,
+                skipped: 0,
+                errors: totalErrors,
+                duration: endTime - startTime
+            }));
+            // Handle transaction commit/rollback
+            if (!flags['dry-run'] && hasActiveTransaction) {
+                const dataProvider = core_2.Metadata.Provider;
+                // We know we have an active transaction at this point
+                if (dataProvider) {
+                    let shouldCommit = true;
+                    // If there are any errors, always rollback
+                    if (totalErrors > 0 || this.errors.length > 0) {
+                        shouldCommit = false;
+                        this.log('\n❌ Errors detected - rolling back all changes');
+                    }
+                    // If there are warnings, ask user (unless in CI mode)
+                    else if (this.warnings.length > 0) {
+                        // Filter out transaction-related warnings since we're now using transactions
+                        const nonTransactionWarnings = this.warnings.filter(w => !w.includes('Transaction support not available') &&
+                            !w.includes('Failed to start transaction'));
+                        if (nonTransactionWarnings.length > 0) {
+                            if (flags.ci) {
+                                // In CI mode, rollback on warnings
+                                shouldCommit = false;
+                                this.log('\n⚠️  Warnings detected in CI mode - rolling back all changes');
+                            }
+                            else {
+                                // Show warnings to user
+                                this.log('\n⚠️  The following warnings were encountered:');
+                                for (const warning of nonTransactionWarnings) {
+                                    this.log(`   - ${warning}`);
+                                }
+                                // Ask user whether to commit or rollback
+                                shouldCommit = await (0, prompts_1.confirm)({
+                                    message: 'Do you want to commit these changes despite the warnings?',
+                                    default: false // Default to rollback
+                                });
+                            }
+                        }
+                    }
+                    try {
+                        if (shouldCommit) {
+                            await dataProvider.CommitTransaction();
+                            this.log('\n✅ All changes committed successfully');
+                            // Clean up file backups after successful commit
+                            await fileBackupManager.cleanup();
+                        }
+                        else {
+                            // User chose to rollback or errors/warnings in CI mode
+                            this.log('\n🔙 Rolling back all changes...');
+                            // Rollback database transaction
+                            await dataProvider.RollbackTransaction();
+                            // Rollback file changes
+                            this.log('🔙 Rolling back file changes...');
+                            await fileBackupManager.rollback();
+                            this.log('✅ Rollback completed - no changes were made to the database or files');
+                        }
+                    }
+                    catch (error) {
+                        // Try to rollback on any error
+                        this.log('\n❌ Transaction error - attempting to roll back changes');
+                        try {
+                            await dataProvider.RollbackTransaction();
+                            this.log('✅ Database rollback completed');
+                        }
+                        catch (rollbackError) {
+                            this.log('❌ Database rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
+                        }
+                        // Also rollback file changes
+                        try {
+                            this.log('🔙 Rolling back file changes...');
+                            await fileBackupManager.rollback();
+                            this.log('✅ File rollback completed');
+                        }
+                        catch (fileRollbackError) {
+                            this.log('❌ File rollback failed: ' + (fileRollbackError instanceof Error ? fileRollbackError.message : String(fileRollbackError)));
+                        }
+                        throw error;
+                    }
+                }
+            }
+            // Exit with error if there were errors in CI mode
+            if ((totalErrors > 0 || this.errors.length > 0 || (this.warnings.length > 0 && flags.ci)) && flags.ci) {
+                this.error('Push failed in CI mode');
             }
         }
         catch (error) {
             spinner.fail('Push failed');
+            // Try to rollback the transaction and files if not in dry-run mode
+            if (!flags['dry-run']) {
+                const { getDataProvider } = await Promise.resolve().then(() => __importStar(require('../../lib/provider-utils')));
+                const dataProvider = getDataProvider();
+                // Rollback database transaction if we have one
+                if (hasActiveTransaction && dataProvider && typeof dataProvider.RollbackTransaction === 'function') {
+                    try {
+                        this.log('\n🔙 Rolling back database transaction due to error...');
+                        await dataProvider.RollbackTransaction();
+                        this.log('✅ Database rollback completed');
+                    }
+                    catch (rollbackError) {
+                        this.log('❌ Database rollback failed: ' + (rollbackError instanceof Error ? rollbackError.message : String(rollbackError)));
+                    }
+                }
+                // Rollback file changes
+                try {
+                    this.log('🔙 Rolling back file changes...');
+                    await fileBackupManager.rollback();
+                    this.log('✅ File rollback completed - all files restored to original state');
+                }
+                catch (fileRollbackError) {
+                    this.log('❌ File rollback failed: ' + (fileRollbackError instanceof Error ? fileRollbackError.message : String(fileRollbackError)));
+                }
+            }
             // Enhanced error logging for debugging
             this.log('\n=== Push Error Details ===');
             this.log(`Error type: ${error?.constructor?.name || 'Unknown'}`);
@@ -176,7 +429,9 @@ class Push extends core_1.Command {
             if (sqlLogger) {
                 try {
                     await sqlLogger.dispose();
-                    this.log('✅ SQL logging session closed');
+                    if (flags.verbose) {
+                        this.log('✅ SQL logging session closed');
+                    }
                 }
                 catch (error) {
                     this.warn(`Failed to close SQL logging session: ${error}`);
@@ -184,14 +439,13 @@ class Push extends core_1.Command {
             }
             // Reset sync engine singleton
             (0, singleton_manager_1.resetSyncEngine)();
-            // Clean up database connection
-            await (0, provider_utils_2.cleanupProvider)();
             // Exit process to prevent background MJ tasks from throwing errors
+            // We don't explicitly close the connection - let the process termination handle it
             process.exit(0);
         }
     }
-    async processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig) {
-        const result = { created: 0, updated: 0, errors: 0 };
+    async processEntityDirectory(entityDir, entityConfig, syncEngine, flags, syncConfig, fileBackupManager) {
+        const result = { created: 0, updated: 0, unchanged: 0, errors: 0 };
         // Find files matching the configured pattern
         const pattern = entityConfig.filePattern || '*.json';
         const jsonFiles = await (0, fast_glob_1.default)(pattern, {
@@ -200,9 +454,11 @@ class Push extends core_1.Command {
             dot: true, // Include dotfiles (files starting with .)
             onlyFiles: true
         });
-        this.log(`Processing ${jsonFiles.length} records in ${path_1.default.relative(process.cwd(), entityDir) || '.'}`);
+        if (flags.verbose) {
+            this.log(`Processing ${jsonFiles.length} records in ${path_1.default.relative(process.cwd(), entityDir) || '.'}`);
+        }
         // First, process all JSON files in this directory
-        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result);
+        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager);
         // Then, recursively process subdirectories
         const entries = await fs_extra_1.default.readdir(entityDir, { withFileTypes: true });
         for (const entry of entries) {
@@ -228,47 +484,68 @@ class Push extends core_1.Command {
                     };
                 }
                 // Process subdirectory with merged config
-                const subResult = await this.processEntityDirectory(subDir, subEntityConfig, syncEngine, flags, syncConfig);
+                const subResult = await this.processEntityDirectory(subDir, subEntityConfig, syncEngine, flags, syncConfig, fileBackupManager);
                 result.created += subResult.created;
                 result.updated += subResult.updated;
+                result.unchanged += subResult.unchanged;
                 result.errors += subResult.errors;
             }
         }
         return result;
     }
-    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result) {
+    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager) {
         if (jsonFiles.length === 0) {
             return;
         }
         const spinner = (0, ora_classic_1.default)();
         spinner.start('Processing records');
-        let totalRecords = 0;
         for (const file of jsonFiles) {
             try {
                 const filePath = path_1.default.join(entityDir, file);
-                const fileContent = await fs_extra_1.default.readJson(filePath);
+                // Backup the file before any modifications (unless dry-run)
+                if (!flags['dry-run'] && fileBackupManager) {
+                    await fileBackupManager.backupFile(filePath);
+                }
+                // Parse JSON with line number tracking
+                const { content: fileContent, lineNumbers } = await this.parseJsonWithLineNumbers(filePath);
                 // Process templates in the loaded content
                 const processedContent = await syncEngine.processTemplates(fileContent, entityDir);
                 // Check if the file contains a single record or an array of records
                 const isArray = Array.isArray(processedContent);
                 const records = isArray ? processedContent : [processedContent];
-                totalRecords += records.length;
                 // Build and process defaults (including lookups)
                 const defaults = await syncEngine.buildDefaults(filePath, entityConfig);
                 // Process each record in the file
                 for (let i = 0; i < records.length; i++) {
                     const recordData = records[i];
                     // Process the record
-                    const isNew = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined);
+                    const recordLineNumber = lineNumbers.get(i); // Get line number for this array index
+                    const pushResult = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined, fileBackupManager, recordLineNumber);
                     if (!flags['dry-run']) {
-                        if (isNew) {
-                            result.created++;
+                        // Don't count duplicates in stats
+                        if (!pushResult.isDuplicate) {
+                            if (pushResult.isNew) {
+                                result.created++;
+                            }
+                            else if (pushResult.wasActuallyUpdated) {
+                                result.updated++;
+                            }
+                            else {
+                                result.unchanged++;
+                            }
                         }
-                        else {
-                            result.updated++;
+                        // Add related entity stats
+                        if (pushResult.relatedStats) {
+                            result.created += pushResult.relatedStats.created;
+                            result.updated += pushResult.relatedStats.updated;
+                            result.unchanged += pushResult.relatedStats.unchanged;
+                            // Debug logging for related entities
+                            if (flags.verbose && pushResult.relatedStats.unchanged > 0) {
+                                this.log(`   Related entities: ${pushResult.relatedStats.unchanged} unchanged`);
+                            }
                         }
                     }
-                    spinner.text = `Processing records (${result.created + result.updated + result.errors}/${totalRecords})`;
+                    spinner.text = `Processing records (${result.created + result.updated + result.unchanged + result.errors} processed)`;
                 }
                 // Write back the entire file if it's an array
                 if (isArray && !flags['dry-run']) {
@@ -278,17 +555,45 @@ class Push extends core_1.Command {
             catch (error) {
                 result.errors++;
                 const errorMessage = error instanceof Error ? error.message : String(error);
-                this.error(`Failed to process ${file}: ${errorMessage}`, { exit: false });
+                const fullErrorMessage = `Failed to process ${file}: ${errorMessage}`;
+                this.errors.push(fullErrorMessage);
+                this.error(fullErrorMessage, { exit: false });
             }
         }
-        spinner.succeed(`Processed ${totalRecords} records from ${jsonFiles.length} files`);
+        if (flags.verbose) {
+            spinner.succeed(`Processed ${result.created + result.updated + result.unchanged} records from ${jsonFiles.length} files`);
+        }
+        else {
+            spinner.stop();
+        }
     }
-    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex) {
+    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex, fileBackupManager, lineNumber) {
         // Load or create entity
         let entity = null;
         let isNew = false;
         if (recordData.primaryKey) {
             entity = await syncEngine.loadEntity(entityName, recordData.primaryKey);
+            // Warn if record has primaryKey but wasn't found
+            if (!entity) {
+                const pkDisplay = Object.entries(recordData.primaryKey)
+                    .map(([key, value]) => `${key}=${value}`)
+                    .join(', ');
+                // Load sync config to check autoCreateMissingRecords setting
+                const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
+                if (!autoCreate) {
+                    const fileRef = lineNumber ? `${fileName}:${lineNumber}` : fileName;
+                    this.warn(`⚠️  Record not found: ${entityName} with primaryKey {${pkDisplay}} at ${fileRef}`);
+                    this.warn(`   To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`);
+                    // Skip this record
+                    return { isNew: false, wasActuallyUpdated: false, isDuplicate: false };
+                }
+                else {
+                    if (verbose) {
+                        this.log(`   Auto-creating missing ${entityName} record with primaryKey {${pkDisplay}}`);
+                    }
+                }
+            }
         }
         if (!entity) {
             // New record
@@ -349,7 +654,46 @@ class Push extends core_1.Command {
         }
         if (dryRun) {
             this.log(`Would ${isNew ? 'create' : 'update'} ${entityName} record`);
-            return isNew;
+            return { isNew, wasActuallyUpdated: true, isDuplicate: false, relatedStats: undefined };
+        }
+        // Check for duplicate processing (but only for existing records that were loaded)
+        let isDuplicate = false;
+        if (!isNew && entity) {
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            isDuplicate = this.checkAndTrackRecord(entityName, entity, fullFilePath, arrayIndex, lineNumber);
+        }
+        // Check if the record is dirty before saving
+        let wasActuallyUpdated = false;
+        if (!isNew && entity.Dirty) {
+            // Record is dirty, get the changes
+            const changes = entity.GetChangesSinceLastSave();
+            const changeKeys = Object.keys(changes);
+            if (changeKeys.length > 0) {
+                wasActuallyUpdated = true;
+                // Get primary key info for display
+                const entityInfo = syncEngine.getEntityInfo(entityName);
+                const primaryKeyDisplay = [];
+                if (entityInfo) {
+                    for (const pk of entityInfo.PrimaryKeys) {
+                        primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                    }
+                }
+                this.log(''); // Add newline before update output
+                this.log(`📝 Updating ${entityName} record:`);
+                if (primaryKeyDisplay.length > 0) {
+                    this.log(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+                }
+                this.log(`   Changes:`);
+                for (const fieldName of changeKeys) {
+                    const field = entity.GetFieldByName(fieldName);
+                    const oldValue = field ? field.OldValue : undefined;
+                    const newValue = changes[fieldName];
+                    this.log(`     ${fieldName}: ${oldValue} → ${newValue}`);
+                }
+            }
+        }
+        else if (isNew) {
+            wasActuallyUpdated = true;
         }
         // Save the record
         const saved = await entity.Save();
@@ -362,9 +706,12 @@ class Push extends core_1.Command {
             throw new Error(`Failed to save record: ${errors}`);
         }
         // Process related entities after saving parent
+        let relatedStats;
         if (recordData.relatedEntities && !dryRun) {
-            await this.processRelatedEntities(recordData.relatedEntities, entity, entity, // root is same as parent for top level
-            baseDir, syncEngine, verbose);
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            relatedStats = await this.processRelatedEntities(recordData.relatedEntities, entity, entity, // root is same as parent for top level
+            baseDir, syncEngine, verbose, fileBackupManager, 1, // indentLevel
+            fullFilePath, arrayIndex);
         }
         // Update the local file with new primary key if created
         if (isNew) {
@@ -376,6 +723,9 @@ class Push extends core_1.Command {
                 }
                 recordData.primaryKey = newPrimaryKey;
             }
+            // Track the new record now that we have its primary key
+            const fullFilePath = path_1.default.join(baseDir, fileName);
+            this.checkAndTrackRecord(entityName, entity, fullFilePath, arrayIndex, lineNumber);
         }
         // Always update sync metadata
         // This ensures related entities are persisted with their metadata
@@ -389,12 +739,15 @@ class Push extends core_1.Command {
             const filePath = path_1.default.join(baseDir, fileName);
             await fs_extra_1.default.writeJson(filePath, recordData, { spaces: 2 });
         }
-        return isNew;
+        return { isNew, wasActuallyUpdated, isDuplicate, relatedStats };
     }
-    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, indentLevel = 1) {
+    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, fileBackupManager, indentLevel = 1, parentFilePath, parentArrayIndex) {
         const indent = '  '.repeat(indentLevel);
+        const stats = { created: 0, updated: 0, unchanged: 0 };
         for (const [entityName, records] of Object.entries(relatedEntities)) {
-            this.log(`${indent}↳ Processing ${records.length} related ${entityName} records`);
+            if (verbose) {
+                this.log(`${indent}↳ Processing ${records.length} related ${entityName} records`);
+            }
             for (const relatedRecord of records) {
                 try {
                     // Load or create entity
@@ -402,6 +755,27 @@ class Push extends core_1.Command {
                     let isNew = false;
                     if (relatedRecord.primaryKey) {
                         entity = await syncEngine.loadEntity(entityName, relatedRecord.primaryKey);
+                        // Warn if record has primaryKey but wasn't found
+                        if (!entity) {
+                            const pkDisplay = Object.entries(relatedRecord.primaryKey)
+                                .map(([key, value]) => `${key}=${value}`)
+                                .join(', ');
+                            // Load sync config to check autoCreateMissingRecords setting
+                            const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                            const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
+                            if (!autoCreate) {
+                                const fileRef = parentFilePath ? path_1.default.relative(config_manager_1.configManager.getOriginalCwd(), parentFilePath) : 'unknown';
+                                this.warn(`${indent}⚠️  Related record not found: ${entityName} with primaryKey {${pkDisplay}} at ${fileRef}`);
+                                this.warn(`${indent}   To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`);
+                                // Skip this record
+                                continue;
+                            }
+                            else {
+                                if (verbose) {
+                                    this.log(`${indent}   Auto-creating missing related ${entityName} record with primaryKey {${pkDisplay}}`);
+                                }
+                            }
+                        }
                     }
                     if (!entity) {
                         entity = await syncEngine.createEntityObject(entityName);
@@ -452,6 +826,46 @@ class Push extends core_1.Command {
                             this.warn(`${indent}  Field '${field}' does not exist on entity '${entityName}'`);
                         }
                     }
+                    // Check for duplicate processing (but only for existing records that were loaded)
+                    let isDuplicate = false;
+                    if (!isNew && entity) {
+                        // Use parent file path for related entities since they're defined in the parent's file
+                        const relatedFilePath = parentFilePath || path_1.default.join(baseDir, 'unknown');
+                        isDuplicate = this.checkAndTrackRecord(entityName, entity, relatedFilePath, parentArrayIndex);
+                    }
+                    // Check if the record is dirty before saving
+                    let wasActuallyUpdated = false;
+                    if (!isNew && entity.Dirty) {
+                        // Record is dirty, get the changes
+                        const changes = entity.GetChangesSinceLastSave();
+                        const changeKeys = Object.keys(changes);
+                        if (changeKeys.length > 0) {
+                            wasActuallyUpdated = true;
+                            // Get primary key info for display
+                            const entityInfo = syncEngine.getEntityInfo(entityName);
+                            const primaryKeyDisplay = [];
+                            if (entityInfo) {
+                                for (const pk of entityInfo.PrimaryKeys) {
+                                    primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                                }
+                            }
+                            this.log(''); // Add newline before update output
+                            this.log(`${indent}📝 Updating related ${entityName} record:`);
+                            if (primaryKeyDisplay.length > 0) {
+                                this.log(`${indent}   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+                            }
+                            this.log(`${indent}   Changes:`);
+                            for (const fieldName of changeKeys) {
+                                const field = entity.GetFieldByName(fieldName);
+                                const oldValue = field ? field.OldValue : undefined;
+                                const newValue = changes[fieldName];
+                                this.log(`${indent}     ${fieldName}: ${oldValue} → ${newValue}`);
+                            }
+                        }
+                    }
+                    else if (isNew) {
+                        wasActuallyUpdated = true;
+                    }
                     // Save the related entity
                     const saved = await entity.Save();
                     if (!saved) {
@@ -462,9 +876,24 @@ class Push extends core_1.Command {
                         const errors = entity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || 'Unknown error';
                         throw new Error(`Failed to save related ${entityName}: ${errors}`);
                     }
-                    if (verbose) {
+                    // Update stats - don't count duplicates
+                    if (!isDuplicate) {
+                        if (isNew) {
+                            stats.created++;
+                        }
+                        else if (wasActuallyUpdated) {
+                            stats.updated++;
+                        }
+                        else {
+                            stats.unchanged++;
+                        }
+                    }
+                    if (verbose && wasActuallyUpdated) {
                         this.log(`${indent}  ✓ ${isNew ? 'Created' : 'Updated'} ${entityName} record`);
                     }
+                    else if (verbose && !wasActuallyUpdated) {
+                        this.log(`${indent}  - No changes to ${entityName} record`);
+                    }
                     // Update the related record with primary key and sync metadata
                     const entityInfo = syncEngine.getEntityInfo(entityName);
                     if (entityInfo) {
@@ -474,6 +903,9 @@ class Push extends core_1.Command {
                             for (const pk of entityInfo.PrimaryKeys) {
                                 relatedRecord.primaryKey[pk.Name] = entity.Get(pk.Name);
                             }
+                            // Track the new related entity now that we have its primary key
+                            const relatedFilePath = parentFilePath || path_1.default.join(baseDir, 'unknown');
+                            this.checkAndTrackRecord(entityName, entity, relatedFilePath, parentArrayIndex);
                         }
                         // Always update sync metadata
                         relatedRecord.sync = {
@@ -483,7 +915,11 @@ class Push extends core_1.Command {
                     }
                     // Process nested related entities if any
                     if (relatedRecord.relatedEntities) {
-                        await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, indentLevel + 1);
+                        const nestedStats = await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, fileBackupManager, indentLevel + 1, parentFilePath, parentArrayIndex);
+                        // Accumulate nested stats
+                        stats.created += nestedStats.created;
+                        stats.updated += nestedStats.updated;
+                        stats.unchanged += nestedStats.unchanged;
                     }
                 }
                 catch (error) {
@@ -491,6 +927,114 @@ class Push extends core_1.Command {
                 }
             }
         }
+        return stats;
+    }
+    /**
+     * Generate a unique tracking key for a record based on entity name and primary key values
+     */
+    generateRecordKey(entityName, entity) {
+        const entityInfo = entity.EntityInfo;
+        const primaryKeyValues = [];
+        if (entityInfo && entityInfo.PrimaryKeys.length > 0) {
+            for (const pk of entityInfo.PrimaryKeys) {
+                const value = entity.Get(pk.Name);
+                primaryKeyValues.push(`${pk.Name}:${value}`);
+            }
+        }
+        return `${entityName}|${primaryKeyValues.join('|')}`;
+    }
+    /**
+     * Check if a record has already been processed and warn if duplicate
+     */
+    checkAndTrackRecord(entityName, entity, filePath, arrayIndex, lineNumber) {
+        const recordKey = this.generateRecordKey(entityName, entity);
+        const existing = this.processedRecords.get(recordKey);
+        if (existing) {
+            const primaryKeyDisplay = entity.EntityInfo?.PrimaryKeys
+                .map(pk => `${pk.Name}: ${entity.Get(pk.Name)}`)
+                .join(', ') || 'unknown';
+            // Format file location with clickable link for VSCode
+            // Create maps with just the line numbers we have
+            const currentLineMap = lineNumber ? new Map([[arrayIndex || 0, lineNumber]]) : undefined;
+            const originalLineMap = existing.lineNumber ? new Map([[existing.arrayIndex || 0, existing.lineNumber]]) : undefined;
+            const currentLocation = this.formatFileLocation(filePath, arrayIndex, currentLineMap);
+            const originalLocation = this.formatFileLocation(existing.filePath, existing.arrayIndex, originalLineMap);
+            this.warn(`⚠️  Duplicate record detected for ${entityName} (${primaryKeyDisplay})`);
+            this.warn(`   Current location:  ${currentLocation}`);
+            this.warn(`   Original location: ${originalLocation}`);
+            this.warn(`   The duplicate update will proceed, but you should review your data for unintended duplicates.`);
+            return true; // is duplicate
+        }
+        // Track the record with its source location
+        this.processedRecords.set(recordKey, {
+            filePath: filePath || 'unknown',
+            arrayIndex,
+            lineNumber
+        });
+        return false; // not duplicate
+    }
+    /**
+     * Parse JSON file and track line numbers for array elements
+     */
+    async parseJsonWithLineNumbers(filePath) {
+        const fileText = await fs_extra_1.default.readFile(filePath, 'utf-8');
+        const lines = fileText.split('\n');
+        const lineNumbers = new Map();
+        // Parse the JSON
+        const content = JSON.parse(fileText);
+        // If it's an array, try to find where each element starts
+        if (Array.isArray(content)) {
+            let inString = false;
+            let bracketDepth = 0;
+            let currentIndex = -1;
+            for (let lineNum = 0; lineNum < lines.length; lineNum++) {
+                const line = lines[lineNum];
+                // Simple tracking of string boundaries and bracket depth
+                for (let i = 0; i < line.length; i++) {
+                    const char = line[i];
+                    const prevChar = i > 0 ? line[i - 1] : '';
+                    if (char === '"' && prevChar !== '\\') {
+                        inString = !inString;
+                    }
+                    if (!inString) {
+                        if (char === '{') {
+                            bracketDepth++;
+                            // If we're at depth 1 in the main array, this is a new object
+                            if (bracketDepth === 1 && line.trim().startsWith('{')) {
+                                currentIndex++;
+                                lineNumbers.set(currentIndex, lineNum + 1); // 1-based line numbers
+                            }
+                        }
+                        else if (char === '}') {
+                            bracketDepth--;
+                        }
+                    }
+                }
+            }
+        }
+        return { content, lineNumbers };
+    }
+    /**
+     * Format file location with clickable link for VSCode
+     */
+    formatFileLocation(filePath, arrayIndex, lineNumbers) {
+        if (!filePath || filePath === 'unknown') {
+            return 'unknown';
+        }
+        // Get absolute path for better VSCode integration
+        const absolutePath = path_1.default.resolve(filePath);
+        // Try to get actual line number from our tracking
+        let lineNumber = 1;
+        if (arrayIndex !== undefined && lineNumbers && lineNumbers.has(arrayIndex)) {
+            lineNumber = lineNumbers.get(arrayIndex);
+        }
+        else if (arrayIndex !== undefined) {
+            // Fallback estimation if we don't have actual line numbers
+            lineNumber = 2 + (arrayIndex * 15);
+        }
+        // Create clickable file path for VSCode - format: file:line
+        // VSCode will make this clickable in the terminal
+        return `${absolutePath}:${lineNumber}`;
     }
 }
 exports.default = Push;