@memberjunction/metadata-sync 2.121.0 → 2.122.1

package/dist/services/PushService.d.ts

@@ -23,6 +23,7 @@ export interface PushResult {
     unchanged: number;
     deleted: number;
     skipped: number;
+    deferred: number;
     errors: number;
     warnings: string[];
     sqlLogPath?: string;
@@ -33,6 +34,7 @@ export interface EntityPushResult {
     unchanged: number;
     deleted: number;
     skipped: number;
+    deferred: number;
     errors: number;
 }
 export declare class PushService {
@@ -41,6 +43,7 @@ export declare class PushService {
     private warnings;
     private syncConfig;
     private deferredFileWrites;
+    private deferredRecords;
     constructor(syncEngine: SyncEngine, contextUser: UserInfo);
     push(options: PushOptions, callbacks?: PushCallbacks): Promise<PushResult>;
     private processEntityDirectory;
@@ -65,6 +68,20 @@ export declare class PushService {
      * Recursively check if any records in an array (including nested relatedEntities) have deletions
      */
     private hasAnyDeletions;
+    /**
+     * Process deferred records that had lookup failures during initial processing.
+     * Called in Phase 2.5 after all records are created/updated but before commit.
+     * This handles circular dependencies where records reference each other.
+     *
+     * Re-runs processFlattenedRecord with allowDefer=false, which processes the
+     * entire record exactly as in the initial pass. Now that all records exist,
+     * the lookups should succeed.
+     *
+     * @param options - Push options
+     * @param callbacks - Callbacks for progress/error reporting
+     * @returns Object with created, updated, and errors counts
+     */
+    private processDeferredRecords;
     /**
      * Write all deferred files with updated deletion timestamps
      * Called in Phase 3 after all deletions complete successfully

package/dist/services/PushService.js

@@ -8,6 +8,7 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const fast_glob_1 = __importDefault(require("fast-glob"));
 const core_1 = require("@memberjunction/core");
+const sync_engine_1 = require("../lib/sync-engine");
 const config_1 = require("../config");
 const file_backup_manager_1 = require("../lib/file-backup-manager");
 const config_manager_1 = require("../lib/config-manager");
@@ -27,6 +28,7 @@ class PushService {
     warnings = [];
     syncConfig;
     deferredFileWrites = new Map();
+    deferredRecords = [];
     constructor(syncEngine, contextUser) {
         this.syncEngine = syncEngine;
         this.contextUser = contextUser;
@@ -37,7 +39,9 @@ class PushService {
         if (options.include && options.exclude) {
             throw new Error('Cannot specify both --include and --exclude options. Please use one or the other.');
         }
-        this.deferredFileWrites.clear(); // Reset deferred writes for this push operation
+        // Reset deferred tracking for this push operation
+        this.deferredFileWrites.clear();
+        this.deferredRecords = [];
         const fileBackupManager = new file_backup_manager_1.FileBackupManager();
         // Load sync config for SQL logging settings and autoCreateMissingRecords flag
         // If dir option is specified, load from that directory, otherwise use original CWD
@@ -126,6 +130,7 @@ class PushService {
             let totalUnchanged = 0;
             let totalDeleted = 0;
             let totalSkipped = 0;
+            let totalDeferred = 0;
             let totalErrors = 0;
             // PHASE 0: Audit all deletions across all entities (if any exist)
             let deletionAudit = null;
@@ -164,6 +169,7 @@ class PushService {
                         unchanged: 0,
                         deleted: 0,
                         skipped: 0,
+                        deferred: 0,
                         errors: 0,
                         warnings: this.warnings
                     };
@@ -216,6 +222,9 @@ class PushService {
                         if (result.deleted > 0) {
                             callbacks?.onLog?.(`   ✓ Deleted: ${result.deleted}`);
                         }
+                        if (result.deferred > 0) {
+                            callbacks?.onLog?.(`   ⏳ Deferred: ${result.deferred}`);
+                        }
                         if (result.unchanged > 0) {
                             callbacks?.onLog?.(`   - Unchanged: ${result.unchanged}`);
                         }
@@ -231,6 +240,7 @@ class PushService {
                     totalUnchanged += result.unchanged;
                     totalDeleted += result.deleted;
                     totalSkipped += result.skipped;
+                    totalDeferred += result.deferred;
                     totalErrors += result.errors;
                 }
                 // PHASE 2: Process deletions in reverse dependency order (if any exist)
@@ -239,6 +249,13 @@ class PushService {
                     totalDeleted += deletionResult.deleted;
                     totalErrors += deletionResult.errors;
                 }
+                // PHASE 2.5: Process deferred records (for circular dependencies)
+                if (this.deferredRecords.length > 0 && totalErrors === 0) {
+                    const deferredResult = await this.processDeferredRecords(options, callbacks);
+                    totalCreated += deferredResult.created;
+                    totalUpdated += deferredResult.updated;
+                    totalErrors += deferredResult.errors;
+                }
                 // Commit transaction if successful
                 if (!options.dryRun && totalErrors === 0) {
                     await transactionManager.commitTransaction();
@@ -279,6 +296,7 @@ class PushService {
                 unchanged: totalUnchanged,
                 deleted: totalDeleted,
                 skipped: totalSkipped,
+                deferred: totalDeferred,
                 errors: totalErrors,
                 warnings: this.warnings,
                 sqlLogPath
@@ -313,6 +331,7 @@ class PushService {
         let unchanged = 0;
         let deleted = 0;
         let skipped = 0;
+        let deferred = 0;
         let errors = 0;
         // Find all JSON files in the directory
         const pattern = entityConfig.filePattern || '*.json';
@@ -335,6 +354,9 @@ class PushService {
                 }
                 // Read the raw file data first
                 const rawFileData = await fs_extra_1.default.readJson(filePath);
+                // Keep unprocessed data to write back (preserves @file: references)
+                const unprocessedRecords = Array.isArray(rawFileData) ? rawFileData : [rawFileData];
+                const isArray = Array.isArray(rawFileData);
                 // Only preprocess if there are @include directives
                 let fileData = rawFileData;
                 const jsonString = JSON.stringify(rawFileData);
@@ -346,7 +368,6 @@ class PushService {
                     fileData = await jsonPreprocessor.processFile(filePath);
                 }
                 const records = Array.isArray(fileData) ? fileData : [fileData];
-                const isArray = Array.isArray(fileData);
                 // Analyze dependencies and get sorted records
                 const analyzer = new record_dependency_analyzer_1.RecordDependencyAnalyzer();
                 const analysisResult = await analyzer.analyzeFileRecords(records, entityConfig.entity);
@@ -421,6 +442,10 @@ class PushService {
                                         skipped++;
                                     else if (result.status === 'error')
                                         errors++;
+                                    else if (result.status === 'deferred') {
+                                        created++; // Deferred records were saved (count as created)
+                                        deferred++; // Also track separately for reporting
+                                    }
                                 }
                             }
                         }
@@ -450,6 +475,10 @@ class PushService {
                                         skipped++;
                                     else if (result.status === 'error')
                                         errors++;
+                                    else if (result.status === 'deferred') {
+                                        created++; // Deferred records were saved (count as created)
+                                        deferred++; // Also track separately for reporting
+                                    }
                                 }
                             }
                         }
@@ -463,23 +492,24 @@ class PushService {
                 // Check if this file has any deletion records (including nested relatedEntities)
                 const hasDeletions = this.hasAnyDeletions(records);
                 // Write back to file (handles both single records and arrays)
+                // Use unprocessedRecords to preserve @file: references
                 // Defer writing if file contains deletions - they'll be written after Phase 2
                 if (!options.dryRun) {
                     if (hasDeletions) {
                         // Store for later writing after deletions complete
                         this.deferredFileWrites.set(filePath, {
                             filePath,
-                            records,
+                            records: unprocessedRecords,
                             isArray
                         });
                     }
                     else {
                         // Write immediately for files without deletions
                         if (isArray) {
-                            await json_write_helper_1.JsonWriteHelper.writeOrderedRecordData(filePath, records);
+                            await json_write_helper_1.JsonWriteHelper.writeOrderedRecordData(filePath, unprocessedRecords);
                         }
                         else {
-                            await json_write_helper_1.JsonWriteHelper.writeOrderedRecordData(filePath, records[0]);
+                            await json_write_helper_1.JsonWriteHelper.writeOrderedRecordData(filePath, unprocessedRecords[0]);
                         }
                     }
                 }
@@ -489,9 +519,9 @@ class PushService {
                 throw fileError;
             }
         }
-        return { created, updated, unchanged, deleted, skipped, errors };
+        return { created, updated, unchanged, deleted, skipped, deferred, errors };
     }
-    async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig) {
+    async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, allowDefer = true) {
         const metadata = new core_1.Metadata();
         const { record, entityName, parentContext, id: recordId } = flattenedRecord;
         // Skip deletion records - they're handled in Phase 2
@@ -585,6 +615,8 @@ class PushService {
         }
         // Process field values with parent context and batch context
         // Process each field with better error reporting
+        // Track if we hit any deferrable lookup errors
+        let hasDeferrableLookupError = false;
         for (const [fieldName, fieldValue] of Object.entries(record.fields)) {
             try {
                 const processedValue = await this.syncEngine.processFieldValue(fieldValue, entityDir, parentEntity, null, // rootRecord
@@ -593,7 +625,25 @@ class PushService {
                 entity.Set(fieldName, processedValue);
             }
             catch (fieldError) {
-                // Enhanced error reporting for field processing failures
+                // Check if this is a deferrable lookup error first
+                if (fieldError instanceof sync_engine_1.DeferrableLookupError) {
+                    // If allowDefer is false, we're in deferred processing mode - can't defer again
+                    if (!allowDefer) {
+                        const err = fieldError;
+                        throw new Error(`Deferred lookup still failed: ${err.message}`);
+                    }
+                    // Mark that we need to defer this entire record
+                    hasDeferrableLookupError = true;
+                    // Log that we're deferring this lookup
+                    if (options.verbose) {
+                        callbacks?.onLog?.(`   ⏳ Deferring lookup for ${entityName}.${fieldName} -> ${fieldError.entityName}`);
+                    }
+                    // Don't set this field - continue to try other fields
+                    // We'll re-process the entire record later
+                    continue;
+                }
+                // For other errors, use enhanced error reporting
+                const err = fieldError;
                 const primaryKeyInfo = record.primaryKey ? JSON.stringify(record.primaryKey) : 'NEW';
                 // Helper to log to both console and callbacks
                 const logError = (msg) => {
@@ -601,44 +651,47 @@ class PushService {
                     callbacks?.onLog?.(msg);
                 };
                 // Check if this is a lookup failure
-                if (fieldError.message?.includes('Lookup failed:')) {
+                if (err.message?.includes('Lookup failed:')) {
                     logError(`\n❌ LOOKUP FAILURE in ${entityName} (${primaryKeyInfo})`);
                     logError(`   Field: ${fieldName}`);
                     logError(`   Value: ${fieldValue}`);
-                    logError(`   Error: ${fieldError.message}`);
+                    logError(`   Error: ${err.message}`);
                     logError(`   Tip: Check if the referenced record exists in the target entity\n`);
                 }
-                else if (fieldError.message?.includes('Entity not found:')) {
+                else if (err.message?.includes('Entity not found:')) {
                     logError(`\n❌ ENTITY NOT FOUND in ${entityName} (${primaryKeyInfo})`);
                     logError(`   Field: ${fieldName}`);
                     logError(`   Value: ${fieldValue}`);
-                    logError(`   Error: ${fieldError.message}`);
+                    logError(`   Error: ${err.message}`);
                     logError(`   Tip: Check if the entity name is spelled correctly\n`);
                 }
-                else if (fieldError.message?.includes('Field') && fieldError.message?.includes('not found')) {
+                else if (err.message?.includes('Field') && err.message?.includes('not found')) {
                     logError(`\n❌ FIELD NOT FOUND in ${entityName} (${primaryKeyInfo})`);
                     logError(`   Field: ${fieldName}`);
                     logError(`   Value: ${fieldValue}`);
-                    logError(`   Error: ${fieldError.message}`);
+                    logError(`   Error: ${err.message}`);
                     logError(`   Tip: Check if the field name exists in the target entity\n`);
                 }
-                else if (fieldError.message?.includes('File not found:')) {
+                else if (err.message?.includes('File not found:')) {
                     logError(`\n❌ FILE NOT FOUND in ${entityName} (${primaryKeyInfo})`);
                     logError(`   Field: ${fieldName}`);
                     logError(`   Value: ${fieldValue}`);
-                    logError(`   Error: ${fieldError.message}`);
+                    logError(`   Error: ${err.message}`);
                     logError(`   Tip: Check if the file path is correct relative to ${entityDir}\n`);
                 }
                 else {
                     logError(`\n❌ FIELD PROCESSING ERROR in ${entityName} (${primaryKeyInfo})`);
                     logError(`   Field: ${fieldName}`);
                     logError(`   Value: ${fieldValue}`);
-                    logError(`   Error: ${fieldError.message}\n`);
+                    logError(`   Error: ${err.message}\n`);
                 }
                 // Re-throw with enhanced context
-                throw new Error(`Failed to process field '${fieldName}' in ${entityName}: ${fieldError.message}`);
+                throw new Error(`Failed to process field '${fieldName}' in ${entityName}: ${err.message}`);
             }
         }
+        // Note: If we had deferred fields, we still continue to save the record
+        // The deferred fields are not set, but other fields are. We'll queue for
+        // re-processing after save succeeds.
         // Check if the record is actually dirty before considering it changed
         let isDirty = entity.Dirty;
         // Force dirty state if alwaysPush is enabled
@@ -826,6 +879,21 @@ class PushService {
         // Add to batch context AFTER save so it has an ID for child @parent:ID references
         // Use the recordId (lookupKey) as the key so child records can find this parent
         batchContext.set(lookupKey, entity);
+        // If we had deferred lookup errors, queue the entire record for re-processing
+        // The record has been saved (without the deferred fields), so it exists in the DB.
+        // In Phase 2.5, we'll re-run processFlattenedRecord with allowDefer=false to fill in the gaps.
+        if (hasDeferrableLookupError && allowDefer && entityConfig) {
+            this.deferredRecords.push({
+                flattenedRecord,
+                entityDir,
+                entityConfig
+            });
+            if (options.verbose) {
+                callbacks?.onLog?.(`   📋 Queued ${entityName} for deferred processing (record saved, some fields pending)`);
+            }
+            // Return 'deferred' status - it's saved but incomplete
+            // We don't return early here because we still want to update primaryKey and sync metadata
+        }
         // Update primaryKey for new records
         if (isNew) {
             const entityInfo = this.syncEngine.getEntityInfo(entityName);
@@ -852,6 +920,15 @@ class PushService {
         }
         // Restore original field values to preserve @ references
         record.fields = originalFields;
+        // Return appropriate status
+        // If we had deferred lookups, return 'deferred' to indicate partial save
+        // The record is saved but will be re-processed in Phase 2.5
+        if (hasDeferrableLookupError && allowDefer) {
+            return {
+                status: 'deferred',
+                isDuplicate: false
+            };
+        }
         return {
             status: isNew ? 'created' : (isDirty ? 'updated' : 'unchanged'),
             isDuplicate: false
@@ -1212,6 +1289,73 @@ class PushService {
         }
         return false;
     }
+    /**
+     * Process deferred records that had lookup failures during initial processing.
+     * Called in Phase 2.5 after all records are created/updated but before commit.
+     * This handles circular dependencies where records reference each other.
+     *
+     * Re-runs processFlattenedRecord with allowDefer=false, which processes the
+     * entire record exactly as in the initial pass. Now that all records exist,
+     * the lookups should succeed.
+     *
+     * @param options - Push options
+     * @param callbacks - Callbacks for progress/error reporting
+     * @returns Object with created, updated, and errors counts
+     */
+    async processDeferredRecords(options, callbacks) {
+        if (this.deferredRecords.length === 0) {
+            return { created: 0, updated: 0, errors: 0 };
+        }
+        callbacks?.onLog?.(`\n⏳ Processing ${this.deferredRecords.length} deferred record${this.deferredRecords.length > 1 ? 's' : ''}...`);
+        let created = 0;
+        let updated = 0;
+        let errors = 0;
+        // Create a fresh batch context for deferred processing
+        // Records are in DB now, so this is mainly for tracking within this phase
+        const batchContext = new Map();
+        for (const deferred of this.deferredRecords) {
+            const { flattenedRecord, entityDir, entityConfig } = deferred;
+            const entityName = flattenedRecord.entityName;
+            const recordId = flattenedRecord.record.primaryKey
+                ? Object.entries(flattenedRecord.record.primaryKey).map(([k, v]) => `${k}=${v}`).join(', ')
+                : (flattenedRecord.record.fields.Name || 'NEW');
+            try {
+                // Re-run processFlattenedRecord with allowDefer=false
+                // This ensures we use the exact same processing logic
+                const result = await this.processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, false // allowDefer=false - must succeed or fail, no re-deferring
+                );
+                if (result.status === 'created') {
+                    created++;
+                    callbacks?.onLog?.(`   ✓ ${entityName} (${recordId}) - created`);
+                }
+                else if (result.status === 'updated') {
+                    updated++;
+                    callbacks?.onLog?.(`   ✓ ${entityName} (${recordId}) - updated`);
+                }
+                else if (result.status === 'unchanged') {
+                    callbacks?.onLog?.(`   - ${entityName} (${recordId}) - unchanged`);
+                }
+            }
+            catch (error) {
+                const err = error;
+                callbacks?.onError?.(`   ✗ Failed to process deferred record: ${entityName} (${recordId})`);
+                callbacks?.onError?.(`     Error: ${err.message}`);
+                callbacks?.onError?.(`     Tip: Ensure all referenced records exist or remove the ?allowDefer flag`);
+                errors++;
+            }
+        }
+        // Summary
+        callbacks?.onLog?.('');
+        const total = created + updated;
+        if (total > 0) {
+            callbacks?.onLog?.(`   ✓ Resolved ${total} deferred record${total > 1 ? 's' : ''} (${created} created, ${updated} updated)`);
+        }
+        if (errors > 0) {
+            callbacks?.onLog?.(`   ✗ Failed to resolve ${errors} deferred record${errors > 1 ? 's' : ''}`);
+        }
+        callbacks?.onLog?.('');
+        return { created, updated, errors };
+    }
     /**
      * Write all deferred files with updated deletion timestamps
      * Called in Phase 3 after all deletions complete successfully