@memberjunction/metadata-sync 5.30.1 → 5.32.0

package/dist/services/PushService.js

@@ -3,6 +3,7 @@ import path from 'path';
 import fastGlob from 'fast-glob';
 import { Metadata } from '@memberjunction/core';
 import { DeferrableLookupError } from '../lib/sync-engine.js';
+import { BatchContextIndex } from '../lib/batch-context-index.js';
 import { loadEntityConfig, loadSyncConfig } from '../config.js';
 import { FileBackupManager } from '../lib/file-backup-manager.js';
 import { configManager } from '../lib/config-manager.js';
@@ -14,16 +15,27 @@ import { JsonPreprocessor } from '../lib/json-preprocessor.js';
 import { findEntityDirectories } from '../lib/provider-utils.js';
 import { DeletionAuditor } from '../lib/deletion-auditor.js';
 import { DeletionReportGenerator } from '../lib/deletion-report-generator.js';
-// Configuration for parallel processing
-const PARALLEL_BATCH_SIZE = 1; // Number of records to process in parallel at each dependency level
+// Configuration for parallel processing.
+// The side-effect-as-data pattern (processFlattenedRecord returns mutations instead of
+// mutating shared state) makes parallel execution safe from a sync-engine perspective.
+// However, entity Save() overrides (e.g., MJActionEntityServer, MJAIPromptEntityServer)
+// may start transactions, do check-then-create patterns, or interact with shared singletons
+// that assume sequential execution. Default stays at 1 for safety; users can opt in to
+// higher values via --parallel-batch-size after verifying their entity subclasses are safe.
+const PARALLEL_BATCH_SIZE = 1;
 export class PushService {
-    constructor(syncEngine, contextUser) {
+    constructor(syncEngine, contextUser, stateManager) {
         this.warnings = [];
         this.syncConfig = null;
         this.deferredFileWrites = new Map();
         this.deferredRecords = [];
         this.syncEngine = syncEngine;
         this.contextUser = contextUser;
+        this.stateManager = stateManager;
+    }
+    /** Set or replace the state manager after construction. */
+    setStateManager(stateManager) {
+        this.stateManager = stateManager;
     }
     /**
      * Determines whether to emit __mj_sync_notes based on config hierarchy.
@@ -77,7 +89,7 @@ export class PushService {
         try {
             // Initialize SQL logger if enabled and not dry-run
             if (sqlLogger.enabled && !options.dryRun) {
-                const provider = Metadata.Provider;
+                const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
                 if (options.verbose) {
                     callbacks?.onLog?.(`SQL logging enabled: ${sqlLogger.enabled}`);
                     callbacks?.onLog?.(`Provider type: ${provider?.constructor?.name || 'Unknown'}`);
@@ -213,7 +225,7 @@ export class PushService {
                     if (options.verbose && callbacks?.onLog) {
                         callbacks.onLog(`Processing ${entityConfig.entity} in ${entityDir}`);
                     }
-                    const result = await this.processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks);
+                    const result = await this.processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, configDir);
                     // Stop the spinner if we were using onProgress
                     if (callbacks?.onProgress && callbacks?.onSuccess) {
                         callbacks.onSuccess(`Processed ${dirName}`);
@@ -275,7 +287,7 @@ export class PushService {
                 }
             }
             catch (error) {
-                // Rollback transaction on error
+                // Rollback transaction on error.
                 if (!options.dryRun) {
                     callbacks?.onLog?.('\n⚠️  Rolling back database transaction due to error...');
                     await transactionManager.rollbackTransaction();
@@ -334,7 +346,7 @@ export class PushService {
             throw error;
         }
     }
-    async processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks) {
+    async processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, syncRootDir) {
         let created = 0;
         let updated = 0;
         let unchanged = 0;
@@ -366,16 +378,30 @@ export class PushService {
                 // Keep unprocessed data to write back (preserves @file: references)
                 const unprocessedRecords = Array.isArray(rawFileData) ? rawFileData : [rawFileData];
                 const isArray = Array.isArray(rawFileData);
-                // Only preprocess if there are @include directives
+                // Preprocess @include directives before checksumming, so changes to
+                // included files are detected by the incremental skip check.
                 let fileData = rawFileData;
                 const jsonString = JSON.stringify(rawFileData);
                 const hasIncludes = jsonString.includes('"@include"') || jsonString.includes('"@include.');
                 if (hasIncludes) {
-                    // Preprocess the JSON file to handle @include directives
-                    // Create a new preprocessor instance for each file to ensure clean state
                     const jsonPreprocessor = new JsonPreprocessor();
                     fileData = await jsonPreprocessor.processFile(filePath);
                 }
+                // Compute checksum on the RESOLVED content (after @include preprocessing)
+                // so that changes to included files are properly detected.
+                let cachedChecksum;
+                if (options.incremental && this.stateManager && syncRootDir) {
+                    const relativePath = path.relative(syncRootDir, filePath);
+                    cachedChecksum = this.syncEngine.calculateChecksum(fileData);
+                    if (!this.stateManager.hasFileChanged(relativePath, cachedChecksum)) {
+                        const recordCount = Array.isArray(rawFileData) ? rawFileData.length : 1;
+                        if (options.verbose) {
+                            callbacks?.onLog?.(`   Skipping unchanged file: ${path.basename(filePath)}`);
+                        }
+                        unchanged += recordCount;
+                        continue;
+                    }
+                }
                 const records = Array.isArray(fileData) ? fileData : [fileData];
                 // Analyze dependencies and get sorted records
                 const analyzer = new RecordDependencyAnalyzer();
@@ -393,7 +419,7 @@ export class PushService {
                 // Note: While JavaScript is single-threaded, async operations can interleave.
                 // Map operations themselves are atomic, but we ensure records are added to
                 // the context AFTER successful save to maintain consistency.
-                const batchContext = new Map();
+                const batchContext = new BatchContextIndex();
                 // Process records using dependency levels for parallel processing
                 if (analysisResult.dependencyLevels && analysisResult.dependencyLevels.length > 0) {
                     // Use parallel processing with dependency levels
@@ -417,7 +443,8 @@ export class PushService {
                                     return { success: false, error, record: flattenedRecord };
                                 }
                             }));
-                            // Process results and check for errors
+                            // Apply side effects sequentially after Promise.all() resolves.
+                            // This eliminates race conditions from concurrent writes to shared state.
                             for (const batchResult of batchResults) {
                                 if (!batchResult.success) {
                                     // Fail fast on first error with detailed logging
@@ -429,8 +456,18 @@ export class PushService {
                                     // Throw concise error to trigger rollback
                                     throw err;
                                 }
-                                // Update stats for successful results
                                 const result = batchResult.result;
+                                // Apply side effects from the result
+                                if (result.batchContextEntry) {
+                                    batchContext.set(result.batchContextEntry.key, result.batchContextEntry.entity);
+                                }
+                                if (result.deferredRecord) {
+                                    this.deferredRecords.push(result.deferredRecord);
+                                }
+                                if (result.warnings) {
+                                    this.warnings.push(...result.warnings);
+                                }
+                                // Update stats for successful results
                                 // Don't count deletion records - they're counted in Phase 2
                                 if (result.isDeletedRecord) {
                                     continue; // Skip entirely
@@ -465,6 +502,16 @@ export class PushService {
                     for (const flattenedRecord of analysisResult.sortedRecords) {
                         try {
                             const result = await this.processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig);
+                            // Apply side effects (already sequential, but consistent with parallel path)
+                            if (result.batchContextEntry) {
+                                batchContext.set(result.batchContextEntry.key, result.batchContextEntry.entity);
+                            }
+                            if (result.deferredRecord) {
+                                this.deferredRecords.push(result.deferredRecord);
+                            }
+                            if (result.warnings) {
+                                this.warnings.push(...result.warnings);
+                            }
                             // Update stats
                             // Don't count deletion records - they're counted in Phase 2
                             if (!result.isDeletedRecord) {
@@ -522,17 +569,34 @@ export class PushService {
                         }
                     }
                 }
+                // Update stored checksum after successful processing (reuse cached value if available)
+                // Uses resolved content (after @include) so included-file changes are tracked.
+                if (this.stateManager && syncRootDir) {
+                    const relativePath = path.relative(syncRootDir, filePath);
+                    const checksum = cachedChecksum ?? this.syncEngine.calculateChecksum(fileData);
+                    this.stateManager.setFileChecksum(relativePath, checksum);
+                }
             }
             catch (fileError) {
                 // Error details already logged by lower-level handlers, just re-throw
                 throw fileError;
             }
         }
+        // Persist push timestamp for this entity directory after all files processed
+        if (this.stateManager && syncRootDir) {
+            const relativeEntityDir = path.relative(syncRootDir, entityDir);
+            this.stateManager.setLastPushTimestamp(relativeEntityDir, new Date().toISOString());
+            await this.stateManager.pruneStaleChecksums(syncRootDir);
+            await this.stateManager.save();
+        }
         return { created, updated, unchanged, deleted, skipped, deferred, errors };
     }
     async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, allowDefer = true) {
-        const metadata = new Metadata();
+        const metadata = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
         const { record, entityName, parentContext, id: recordId } = flattenedRecord;
+        // Accumulate warnings locally instead of mutating this.warnings directly.
+        // These are returned in the result and applied sequentially after Promise.all().
+        const localWarnings = [];
         // Skip deletion records - they're handled in Phase 2
         // File writing is deferred for files containing deletions
         // Mark with special flag so they don't count in Phase 1 stats at all
@@ -543,11 +607,43 @@ export class PushService {
         // This ensures we can properly find parent entities even when they're new
         const lookupKey = recordId;
         // Check if already in batch context
-        let entity = batchContext.get(lookupKey);
-        if (entity) {
+        const existingEntry = batchContext.get(lookupKey);
+        if (existingEntry) {
             // Already processed
             return { status: 'unchanged', isDuplicate: true };
         }
+        let entity;
+        // Record-level skip (--incremental only): if the record has a stored
+        // checksum and it matches the current fields, nothing changed — skip
+        // without touching the DB. The checksum is embedded in the JSON
+        // (record.sync.checksum) and was stored after the last successful save,
+        // so a match means fields are identical. We still add a lightweight stub
+        // to the batch context so child records at later dependency levels can
+        // resolve @parent:ID references.
+        //
+        // Gated on options.incremental so non-incremental pushes still load the
+        // entity and reassert JSON values over the DB. This preserves the implicit
+        // drift-correction behavior single-env workflows rely on, and prevents
+        // silent no-ops when seeding a fresh DB from JSON files that already
+        // carry stored checksums from another environment.
+        if (options.incremental && record.sync?.checksum && record.fields && record.primaryKey && !record.deleteRecord) {
+            // Use calculateChecksumWithFileContent so @file: reference changes are detected.
+            // This reads local files (fast) rather than hitting the DB (slow). The stored
+            // checksum was also computed with file content, so the comparison is apples-to-apples.
+            const currentChecksum = await this.syncEngine.calculateChecksumWithFileContent(record.fields, entityDir);
+            if (currentChecksum === record.sync.checksum) {
+                // Lightweight stub — just enough for @parent:ID and @lookup resolution.
+                // No DB call, no entity framework overhead.
+                const allFields = { ...record.primaryKey, ...record.fields };
+                const stub = {
+                    EntityInfo: { Name: entityName, PrimaryKeys: Object.keys(record.primaryKey).map(k => ({ Name: k })), Fields: Object.keys(allFields).map(k => ({ Name: k })) },
+                    Get: (field) => allFields[field],
+                    GetAll: () => allFields,
+                };
+                const batchContextEntry = { key: lookupKey, entity: stub };
+                return { status: 'unchanged', isDuplicate: false, batchContextEntry };
+            }
+        }
         // Get or create entity instance
         entity = await metadata.GetEntityObject(entityName, this.contextUser);
         if (!entity) {
@@ -604,9 +700,9 @@ export class PushService {
                     .join(', ');
                 if (!autoCreate) {
                     const warning = `Record not found: ${entityName} with primaryKey {${pkDisplay}}. To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`;
-                    this.warnings.push(warning);
+                    localWarnings.push(warning);
                     callbacks?.onWarn?.(warning);
-                    return { status: 'error', isDuplicate: false }; // This will be counted as error, not skipped
+                    return { status: 'error', isDuplicate: false, warnings: localWarnings }; // This will be counted as error, not skipped
                 }
                 else {
                     // Log that we're creating the missing record
@@ -738,13 +834,17 @@ export class PushService {
             }
         }
         if (options.dryRun) {
+            // Still add to batch context so child records at later dependency levels
+            // can resolve @parent references. Without this, dry-run fails on any
+            // metadata with parent-child nesting (e.g. Actions → Action Params).
+            const batchContextEntry = { key: lookupKey, entity };
             if (exists) {
                 callbacks?.onLog?.(`[DRY RUN] Would update ${entityName} record`);
-                return { status: 'updated' };
+                return { status: 'updated', batchContextEntry };
             }
             else {
                 callbacks?.onLog?.(`[DRY RUN] Would create ${entityName} record`);
-                return { status: 'created' };
+                return { status: 'created', batchContextEntry };
             }
         }
         // If updating an existing record that's dirty, show what changed
@@ -778,6 +878,9 @@ export class PushService {
         const entityRecordId = entity.Get('ID');
         let saveResult;
         try {
+            // Skip embedding generation during sync — vectors can be computed later by the
+            // API server. This avoids loading the ~50MB Xenova model in short-lived CLI processes.
+            entity.SkipEmbeddings = true;
             // Pass IgnoreDirtyState option when alwaysPush is enabled
             const saveOptions = alwaysPush ? { IgnoreDirtyState: true } : undefined;
             saveResult = await entity.Save(saveOptions);
@@ -898,18 +1001,21 @@ export class PushService {
             // Throw error to trigger rollback and stop processing
             throw new Error(`Failed to save ${entityName} record at ${flattenedRecord.path}: ${errorMessage}`);
         }
-        // Add to batch context AFTER save so it has an ID for child @parent:ID references
-        // Use the recordId (lookupKey) as the key so child records can find this parent
-        batchContext.set(lookupKey, entity);
-        // If we had deferred lookup errors, queue the entire record for re-processing
+        // Return batch context entry as a side effect instead of mutating directly.
+        // The caller applies this sequentially after Promise.all() resolves, avoiding
+        // race conditions where concurrent tasks read stale batchContext state.
+        const batchContextEntry = { key: lookupKey, entity };
+        // If we had deferred lookup errors, return the deferred record as a side effect
+        // instead of pushing to this.deferredRecords directly (avoids concurrent pushes).
         // The record has been saved (without the deferred fields), so it exists in the DB.
         // In Phase 2.5, we'll re-run processFlattenedRecord with allowDefer=false to fill in the gaps.
+        let deferredRecord;
         if (hasDeferrableLookupError && allowDefer && entityConfig) {
-            this.deferredRecords.push({
+            deferredRecord = {
                 flattenedRecord,
                 entityDir,
                 entityConfig
-            });
+            };
             if (options.verbose) {
                 callbacks?.onLog?.(`   📋 Queued ${entityName} for deferred processing (record saved, some fields pending)`);
             }
@@ -963,18 +1069,25 @@ export class PushService {
             // emitSyncNotes is disabled - always remove existing notes
             delete recordWithNotes.__mj_sync_notes;
         }
+        // Build result with side effects as data
+        const resultBase = {
+            isDuplicate: false,
+            batchContextEntry,
+            deferredRecord,
+            warnings: localWarnings.length > 0 ? localWarnings : undefined,
+        };
         // Return appropriate status
         // If we had deferred lookups, return 'deferred' to indicate partial save
         // The record is saved but will be re-processed in Phase 2.5
         if (hasDeferrableLookupError && allowDefer) {
             return {
+                ...resultBase,
                 status: 'deferred',
-                isDuplicate: false
             };
         }
         return {
+            ...resultBase,
             status: isNew ? 'created' : (isDirty ? 'updated' : 'unchanged'),
-            isDuplicate: false
         };
     }
     formatFieldValue(value) {
@@ -1266,7 +1379,7 @@ export class PushService {
             callbacks?.onWarn?.(`⚠️  Detected ${analysisResult.circularDependencies.length} circular dependencies across metadata files`);
         }
         // Perform comprehensive deletion audit
-        const md = new Metadata();
+        const md = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
         const auditor = new DeletionAuditor(md, this.contextUser);
         const audit = await auditor.auditDeletions(allRecords, options.deleteDbOnly ?? false);
         // Check if any records actually need deletion
@@ -1413,7 +1526,7 @@ export class PushService {
         let errors = 0;
         // Create a fresh batch context for deferred processing
         // Records are in DB now, so this is mainly for tracking within this phase
-        const batchContext = new Map();
+        const batchContext = new BatchContextIndex();
         for (const deferred of this.deferredRecords) {
             const { flattenedRecord, entityDir, entityConfig } = deferred;
             const entityName = flattenedRecord.entityName;
@@ -1425,6 +1538,14 @@ export class PushService {
                 // This ensures we use the exact same processing logic
                 const result = await this.processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, false // allowDefer=false - must succeed or fail, no re-deferring
                 );
+                // Apply side effects (sequential here, but consistent with parallel path)
+                if (result.batchContextEntry) {
+                    batchContext.set(result.batchContextEntry.key, result.batchContextEntry.entity);
+                }
+                if (result.warnings) {
+                    this.warnings.push(...result.warnings);
+                }
+                // Note: result.deferredRecord should never be set here since allowDefer=false
                 if (result.status === 'created') {
                     created++;
                     callbacks?.onLog?.(`   ✓ ${entityName} (${recordId}) - created`);