@memberjunction/metadata-sync 5.40.2 → 5.42.0

package/dist/services/PushService.js

@@ -1,7 +1,8 @@
 import fs from 'fs-extra';
 import path from 'path';
 import fastGlob from 'fast-glob';
-import { Metadata } from '@memberjunction/core';
+import chalk from 'chalk';
+import { Metadata, IsVerboseLoggingEnabled } from '@memberjunction/core';
 import { UUIDsEqual } from '@memberjunction/global';
 import { IsStringSQLType } from '@memberjunction/sql-dialect';
 import { DeferrableLookupError } from '../lib/sync-engine.js';
@@ -29,6 +30,7 @@ const PARALLEL_BATCH_SIZE = 1;
 export class PushService {
     constructor(syncEngine, contextUser, stateManager) {
         this.warnings = [];
+        this.changeDetails = [];
         this.syncConfig = null;
         this.deferredFileWrites = new Map();
         this.deferredRecords = [];
@@ -73,8 +75,53 @@ export class PushService {
         const t = (sqlType ?? '').trim().toLowerCase();
         return t === 'string' || IsStringSQLType(t);
     }
+    /**
+     * Build the compact, single-line per-directory result shown in normal (non-verbose)
+     * runs — e.g. "ai-models — 722 records, no changes" or
+     * "ai-prompts — 3 created, 1 updated, 8 unchanged". Only non-zero buckets are listed;
+     * a directory with no creates/updates/deletes/errors collapses to "N records, no changes".
+     *
+     * Returns the fully-styled text plus a `changed` flag. No-op directories are dimmed so
+     * the (colored) changed directories pop visually; the `changed` flag lets the caller pick
+     * the success glyph. The "X of N" prefix is dimmed so it reads as secondary.
+     */
+    formatDirectorySummary(prefix, dirName, result, total) {
+        const parts = [];
+        if (result.created > 0)
+            parts.push(chalk.green(`${result.created} created`));
+        if (result.updated > 0)
+            parts.push(chalk.green(`${result.updated} updated`));
+        if (result.deleted > 0)
+            parts.push(chalk.red(`${result.deleted} deleted`));
+        if (result.deferred > 0)
+            parts.push(chalk.yellow(`${result.deferred} deferred`));
+        if (result.skipped > 0)
+            parts.push(chalk.gray(`${result.skipped} skipped`));
+        if (result.errors > 0)
+            parts.push(chalk.red(`${result.errors} error${result.errors === 1 ? '' : 's'}`));
+        const changed = result.created > 0 || result.updated > 0 || result.deleted > 0 || result.deferred > 0 || result.errors > 0;
+        if (parts.length === 0) {
+            // Pure no-op — dim the whole line so genuine changes stand out against the stream.
+            return {
+                text: chalk.dim(`${prefix}${dirName} — ${total} record${total === 1 ? '' : 's'}, no changes`),
+                changed: false,
+            };
+        }
+        // When something changed, append unchanged as trailing context so the totals reconcile.
+        if (result.unchanged > 0)
+            parts.push(chalk.gray(`${result.unchanged} unchanged`));
+        return {
+            text: `${chalk.dim(prefix)}${dirName} — ${parts.join(', ')}`,
+            changed,
+        };
+    }
     async push(options, callbacks) {
         this.warnings = [];
+        this.changeDetails = [];
+        // Respect the global MJ_VERBOSE env/flag in addition to the per-command --verbose
+        // flag, so a single dial (MJ_VERBOSE=1) controls diagnostic verbosity across every
+        // MJ CLI tool (codegen, sync, …) rather than each command inventing its own.
+        options.verbose = options.verbose || IsVerboseLoggingEnabled();
         // Validate that include and exclude are not used together
         if (options.include && options.exclude) {
             throw new Error('Cannot specify both --include and --exclude options. Please use one or the other.');
@@ -162,13 +209,27 @@ export class PushService {
             // Metadata.Provider — `mj sync` is single-process so they resolve to the
             // same instance today, but routing through SyncEngine keeps the wiring
             // self-consistent and makes future provider plumbing trivial.
-            callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
+            // Preload is internal plumbing — emit its progress only in verbose mode so a
+            // normal run jumps straight from validation to per-directory results.
+            if (options.verbose) {
+                callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
+            }
             this.syncMetadataEngine.setEntityDirs(entityDirs);
             await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
             for (const warning of this.syncMetadataEngine.drainWarnings()) {
                 callbacks?.onWarn?.(`   ⚠️  ${warning}`);
             }
-            callbacks?.onLog?.('✓ Preload completed successfully\n');
+            if (options.verbose) {
+                const delegations = this.syncMetadataEngine.getDelegationSummary();
+                if (delegations.length > 0) {
+                    const donorCount = new Set(delegations.map(d => d.engineClassName)).size;
+                    callbacks?.onLog?.(`   ↪ Reused in-memory caches for ${delegations.length} ${delegations.length === 1 ? 'entity' : 'entities'} already loaded by ${donorCount} ${donorCount === 1 ? 'engine' : 'engines'}`);
+                    for (const d of delegations.sort((a, b) => a.entityName.localeCompare(b.entityName))) {
+                        callbacks?.onLog?.(`      • ${d.entityName} ← ${d.engineClassName}`);
+                    }
+                }
+                callbacks?.onLog?.('✓ Preload completed successfully\n');
+            }
             if (options.verbose) {
                 callbacks?.onLog?.(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
             }
@@ -226,7 +287,8 @@ export class PushService {
                         skipped: 0,
                         deferred: 0,
                         errors: 0,
-                        warnings: this.warnings
+                        warnings: this.warnings,
+                        changeLog: this.changeDetails
                     };
                 }
             }
@@ -236,8 +298,13 @@ export class PushService {
             }
             try {
                 // PHASE 1: Process creates/updates for all entities
-                callbacks?.onLog?.('📝 Processing creates and updates...\n');
-                for (const entityDir of entityDirs) {
+                if (options.verbose) {
+                    callbacks?.onLog?.('📝 Processing creates and updates...\n');
+                }
+                for (const [dirIdx, entityDir] of entityDirs.entries()) {
+                    // "X of N" position prefix — only when there's more than one directory, so a
+                    // single-directory push stays uncluttered.
+                    const progressPrefix = entityDirs.length > 1 ? `[${dirIdx + 1}/${entityDirs.length}] ` : '';
                     const entityConfig = await loadEntityConfig(entityDir);
                     if (!entityConfig) {
                         const warning = `Skipping ${entityDir} - no valid entity configuration`;
@@ -246,12 +313,16 @@ export class PushService {
                         totalSkipped++; // Count skipped directories
                         continue;
                     }
-                    // Show folder with spinner at start
+                    // Show folder with spinner at start. The folder header is redundant in a
+                    // normal run (the per-directory result line below names the directory), so
+                    // it's verbose-only; the live spinner still shows "[X/N] Processing <dir>…".
                     const dirName = path.relative(process.cwd(), entityDir) || '.';
-                    callbacks?.onLog?.(`\n📁 ${dirName}:`);
+                    if (options.verbose) {
+                        callbacks?.onLog?.(`\n📁 ${dirName}:`);
+                    }
                     // Use onProgress for animated spinner if available
                     if (callbacks?.onProgress) {
-                        callbacks.onProgress(`Processing ${dirName}...`);
+                        callbacks.onProgress(`${progressPrefix}Processing ${dirName}...`);
                     }
                     else {
                         callbacks?.onLog?.(`   ⏳ Processing...`);
@@ -260,13 +331,18 @@ export class PushService {
                         callbacks.onLog(`Processing ${entityConfig.entity} in ${entityDir}`);
                     }
                     const result = await this.processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, configDir);
-                    // Stop the spinner if we were using onProgress
+                    // Per-directory result: one compact line (always), naming the directory and
+                    // its changes — or "no changes" for a clean dir. The detailed per-status
+                    // breakdown is verbose-only since the final summary box already aggregates it.
+                    const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
+                    const { text: dirSummary, changed: dirChanged } = this.formatDirectorySummary(progressPrefix, dirName, result, dirTotal);
                     if (callbacks?.onProgress && callbacks?.onSuccess) {
-                        callbacks.onSuccess(`Processed ${dirName}`);
+                        callbacks.onSuccess(dirSummary, dirChanged);
                     }
-                    // Show per-directory summary
-                    const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
-                    if (dirTotal > 0 || result.errors > 0) {
+                    else {
+                        callbacks?.onLog?.(`   ${dirSummary}`);
+                    }
+                    if (options.verbose && (dirTotal > 0 || result.errors > 0)) {
                         callbacks?.onLog?.(`   Total processed: ${dirTotal} records`);
                         if (result.created > 0) {
                             callbacks?.onLog?.(`   ✓ Created: ${result.created}`);
@@ -336,13 +412,20 @@ export class PushService {
                     callbacks?.onLog?.('✅ File backups committed');
                 }
             }
-            // Close SQL logging session if it was created
+            // Close SQL logging session if it was created. dispose() deletes the file when
+            // it's empty (no statements emitted), so only surface a saved path when the log
+            // actually has content — otherwise the "SQL log saved to…" line would point at a
+            // file dispose() just unlinked.
             let sqlLogPath;
             if (sqlLoggingSession) {
-                sqlLogPath = sqlLoggingSession.filePath;
+                const filePath = sqlLoggingSession.filePath;
+                const hadStatements = sqlLoggingSession.statementCount > 0;
                 await sqlLoggingSession.dispose();
-                if (options.verbose) {
-                    callbacks?.onLog?.(`📝 SQL log written to: ${sqlLogPath}`);
+                if (hadStatements) {
+                    sqlLogPath = filePath;
+                    if (options.verbose) {
+                        callbacks?.onLog?.(`📝 SQL log written to: ${filePath}`);
+                    }
                 }
             }
             return {
@@ -354,7 +437,8 @@ export class PushService {
                 deferred: totalDeferred,
                 errors: totalErrors,
                 warnings: this.warnings,
-                sqlLogPath
+                sqlLogPath,
+                changeLog: this.changeDetails
             };
         }
         catch (error) {
@@ -748,6 +832,14 @@ export class PushService {
                     const warning = `Record not found: ${entityName} with primaryKey {${pkDisplay}}. To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`;
                     localWarnings.push(warning);
                     callbacks?.onWarn?.(warning);
+                    // Counted as an error below — surface it structurally too (this path
+                    // returns rather than throws, so it wouldn't otherwise reach onRecordError).
+                    callbacks?.onRecordError?.({
+                        entityName,
+                        path: flattenedRecord.path,
+                        primaryKey: pkDisplay,
+                        message: `Record not found: ${entityName} {${pkDisplay}} (set push.autoCreateMissingRecords=true to auto-create)`,
+                    });
                     return { status: 'error', isDuplicate: false, warnings: localWarnings }; // This will be counted as error, not skipped
                 }
                 else {
@@ -933,7 +1025,9 @@ export class PushService {
                 return { status: 'created', batchContextEntry };
             }
         }
-        // If updating an existing record that's dirty, show what changed
+        // If updating an existing record that's dirty, capture what changed for the
+        // end-of-run recap. The inline diff is verbose-only now: the recap is the default
+        // surface, and emitting the diff mid-spinner previously garbled the spinner line.
         if (!isNew && isDirty) {
             const changes = entity.GetChangesSinceLastSave();
             const changeKeys = Object.keys(changes);
@@ -946,16 +1040,29 @@ export class PushService {
                         primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
                     }
                 }
-                callbacks?.onLog?.(`📝 Updating ${entityName} record:`);
-                if (primaryKeyDisplay.length > 0) {
-                    callbacks?.onLog?.(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
-                }
-                callbacks?.onLog?.(`   Changes:`);
-                for (const fieldName of changeKeys) {
+                const fieldDiffs = changeKeys.map((fieldName) => {
                     const field = entity.GetFieldByName(fieldName);
-                    const oldValue = field ? field.OldValue : undefined;
-                    const newValue = changes[fieldName];
-                    callbacks?.onLog?.(`     ${fieldName}: ${this.formatFieldValue(oldValue)} → ${this.formatFieldValue(newValue)}`);
+                    return {
+                        field: fieldName,
+                        oldValue: this.formatFieldValue(field ? field.OldValue : undefined),
+                        newValue: this.formatFieldValue(changes[fieldName]),
+                    };
+                });
+                this.changeDetails.push({
+                    entityName,
+                    primaryKey: primaryKeyDisplay.join(', '),
+                    operation: 'updated',
+                    fields: fieldDiffs,
+                });
+                if (options.verbose) {
+                    callbacks?.onLog?.(`📝 Updating ${entityName} record:`);
+                    if (primaryKeyDisplay.length > 0) {
+                        callbacks?.onLog?.(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+                    }
+                    callbacks?.onLog?.(`   Changes:`);
+                    for (const diff of fieldDiffs) {
+                        callbacks?.onLog?.(`     ${diff.field}: ${diff.oldValue} → ${diff.newValue}`);
+                    }
                 }
             }
         }
@@ -1084,6 +1191,13 @@ export class PushService {
             if (errorMessage.includes('transaction')) {
                 callbacks?.onError?.(`   ⚠️  Transaction error detected. The database transaction may be corrupted.`);
             }
+            // Structured, single-shot error for machine-readable consumers.
+            callbacks?.onRecordError?.({
+                entityName,
+                path: flattenedRecord.path,
+                primaryKey: primaryKeyInfo.length > 0 ? primaryKeyInfo.join(', ') : undefined,
+                message: errorMessage,
+            });
             // Throw error to trigger rollback and stop processing
             throw new Error(`Failed to save ${entityName} record at ${flattenedRecord.path}: ${errorMessage}`);
         }
@@ -1113,10 +1227,19 @@ export class PushService {
             const entityInfo = this.syncEngine.getEntityInfo(entityName);
             if (entityInfo) {
                 const newPrimaryKey = {};
+                const primaryKeyDisplay = [];
                 for (const pk of entityInfo.PrimaryKeys) {
                     newPrimaryKey[pk.Name] = entity.Get(pk.Name);
+                    primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
                 }
                 record.primaryKey = newPrimaryKey;
+                // Capture the create for the changes recap (no field diffs — the whole record is new).
+                this.changeDetails.push({
+                    entityName,
+                    primaryKey: primaryKeyDisplay.join(', '),
+                    operation: 'created',
+                    fields: [],
+                });
             }
         }
         // Only update sync metadata if the record was actually dirty (changed)
@@ -1239,22 +1362,34 @@ export class PushService {
                 primaryKeyDisplay.push(`${pk.Name}: ${existingEntity.Get(pk.Name)}`);
             }
         }
-        if (isDbOnly) {
-            callbacks?.onLog?.(`🗑️  Deleting database-only ${entityName} record:`);
-        }
-        else {
-            callbacks?.onLog?.(`🗑️  Deleting ${entityName} record:`);
-        }
-        if (primaryKeyDisplay.length > 0) {
-            callbacks?.onLog?.(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
-        }
-        // Additional info if available
-        const recordName = existingEntity.Get('Name');
-        if (recordName) {
-            callbacks?.onLog?.(`   Name: ${recordName}`);
+        // Deletion detail streams inline only in verbose mode — the recap is the default
+        // surface (and inline logging mid-spinner garbles the spinner line).
+        if (options.verbose) {
+            if (isDbOnly) {
+                callbacks?.onLog?.(`🗑️  Deleting database-only ${entityName} record:`);
+            }
+            else {
+                callbacks?.onLog?.(`🗑️  Deleting ${entityName} record:`);
+            }
+            if (primaryKeyDisplay.length > 0) {
+                callbacks?.onLog?.(`   Primary Key: ${primaryKeyDisplay.join(', ')}`);
+            }
+            const recordNameVerbose = existingEntity.Get('Name');
+            if (recordNameVerbose) {
+                callbacks?.onLog?.(`   Name: ${recordNameVerbose}`);
+            }
         }
+        // Capture the deletion for the changes recap (no field diffs for deletes).
+        this.changeDetails.push({
+            entityName,
+            primaryKey: primaryKeyDisplay.join(', '),
+            operation: 'deleted',
+            fields: [],
+        });
         if (options.dryRun) {
-            callbacks?.onLog?.(`[DRY RUN] Would delete ${entityName} record`);
+            if (options.verbose) {
+                callbacks?.onLog?.(`[DRY RUN] Would delete ${entityName} record`);
+            }
             return { status: 'deleted', isDuplicate: false };
         }
         // Delete the record
@@ -1270,6 +1405,12 @@ export class PushService {
                 if (errorDetails) {
                     callbacks?.onError?.(`   Details: ${errorDetails}`);
                 }
+                callbacks?.onRecordError?.({
+                    entityName,
+                    path: flattenedRecord.path,
+                    primaryKey: primaryKeyDisplay.length > 0 ? primaryKeyDisplay.join(', ') : undefined,
+                    message: errorMessage,
+                });
                 throw new Error(`Failed to delete ${entityName} record: ${errorMessage}`);
             }
             // Remove from metadata cache