@memberjunction/metadata-sync 6.2.0-edge.4 → 6.2.0-edge.6

package/dist/services/PushService.js

@@ -8,7 +8,7 @@ import { IsStringSQLType } from '@memberjunction/sql-dialect';
 import { SyncEngine, DeferrableLookupError } from '../lib/sync-engine.js';
 import { SyncMetadataEngine } from '../lib/sync-metadata-engine.js';
 import { BatchContextIndex } from '../lib/batch-context-index.js';
-import { loadEntityConfig, loadSyncConfig } from '../config.js';
+import { loadEntityConfig } from '../config.js';
 import { FileBackupManager } from '../lib/file-backup-manager.js';
 import { configManager } from '../lib/config-manager.js';
 import { SQLLogger } from '../lib/sql-logger.js';
@@ -19,10 +19,21 @@ import { GraphProviderPool, ProbeIndependentInstances } from '../lib/graph-provi
 import { ResolveDirectoryMode, GraphBatchSizeFor, IsolatedModeWarning, UnusedBatchSizeWarning, } from '../lib/push-write-mode.js';
 import { PushAbortedError, DescribeCommitFailure, DescribeRollbackOutcome } from '../lib/push-outcome.js';
 import { JsonPreprocessor } from '../lib/json-preprocessor.js';
-import { findEntityDirectories } from '../lib/provider-utils.js';
+import { ResolvePushFolders, SkippedDirectoryMessage } from '../lib/push-directories.js';
 import { DeletionAuditor } from '../lib/deletion-auditor.js';
 import { DescribeMissingEntitySubclass } from '../lib/entity-subclass-guard.js';
 import { DeletionReportGenerator } from '../lib/deletion-report-generator.js';
+import { EnvVariableNotSetError, OptionalEnvMissingError, OptionalEnvSkipMessage, ParseEnvReference } from '../lib/env-reference.js';
+import { RecordStartLine } from '../lib/json-record-lines.js';
+import { EnvDisplayMask, EnvSourceLabel, ENV_HIDDEN_VALUE, MaskedError } from '../lib/env-display.js';
+import { EnvSqlLog, AddSkippedOptionalEnvField } from '../lib/env-sql-redactor.js';
+import { ComposeStatementTransforms, LookupSqlLog } from '../lib/lookup-sql.js';
+import { ResolveSchemaPlaceholders } from '../lib/schema-placeholders.js';
+import { DatabaseNaturalKeyReader, NaturalKeyConflictMessage, NaturalKeyPreflight } from '../lib/natural-key-preflight.js';
+import { CreatedRecordKeyFields, IsKeyReference, KeyFieldValue } from '../lib/created-record-key.js';
+import { EntityFieldName, RecordFieldNames } from '../lib/entity-field-names.js';
+import { METADATA_KEYWORDS } from '../constants/metadata-keywords.js';
+import { DEFAULT_RECORD_FILE_PATTERN, RECORD_FILE_GLOB_OPTIONS } from '../lib/record-file-glob.js';
 import { resolveCollectionRelationship } from '@memberjunction/record-graph';
 function emptyPushTotals() {
     return { created: 0, updated: 0, unchanged: 0, deleted: 0, skipped: 0, deferred: 0, errors: 0 };
@@ -64,11 +75,20 @@ export class PushService {
         this.graphBatchSize = 1;
         /** Counts so far, so a failed push can still report what it did. */
         this.runningTotals = emptyPushTotals();
+        /** The open SQL log's redactor and lookup-create hook; null when no log is open. */
+        this.envSqlLog = null;
+        /** The open SQL log's `@lookup` → subquery rewriter and lookup-create hook; null when no log is open. */
+        this.lookupSqlLog = null;
+        /** Unique keys of each entity's table read for the create pre-flight (#4400), by entity name. */
+        this.uniqueKeyCache = new Map();
+        /** Composition child fields waiting for their owner's save, by the owner record's resolution collector. */
+        this.pendingEnvSqlChildFields = new WeakMap();
         /** Non-atomic mode: creates and updates that were committed outside the push transaction. */
         this.committedWrites = [];
         /** Incremental state, applied only after the push commits. Keyed by path relative to the sync root. */
         this.pendingChecksums = new Map();
         this.pushedEntityDirs = [];
+        this.skippedDirectories = [];
         /** Errors whose record was already sent to onRecordError, so a caller does not report it twice. */
         this.reportedRecordErrors = new WeakSet();
         this.syncEngine = syncEngine;
@@ -179,14 +199,18 @@ export class PushService {
         this.committedWrites = [];
         this.pendingChecksums.clear();
         this.pushedEntityDirs = [];
+        this.skippedDirectories = [];
         this.directoryModes.clear();
         this.runningTotals = emptyPushTotals();
         this.sqlLogFilePath = undefined;
+        this.closeEnvSqlRedactor();
         const fileBackupManager = new FileBackupManager();
         // Load sync config for SQL logging settings and autoCreateMissingRecords flag
-        // If dir option is specified, load from that directory, otherwise use original CWD
-        const configDir = options.dir ? path.resolve(configManager.getOriginalCwd(), options.dir) : configManager.getOriginalCwd();
-        this.syncConfig = await loadSyncConfig(configDir);
+        // With the dir option, load from the sync root at or above that directory, otherwise from the original CWD
+        const folders = await ResolvePushFolders(configManager.getOriginalCwd(), options.dir, options.include, options.exclude);
+        const configDir = folders.ConfigDir;
+        const syncRootDir = folders.SyncRootDir;
+        this.syncConfig = folders.SyncConfig;
         // Display warnings for special flags that are enabled
         if (this.syncConfig?.push?.alwaysPush && !options.dryRun) {
             callbacks?.onWarn?.('\n⚡ WARNING: alwaysPush is enabled - ALL records will be saved to database regardless of changes\n');
@@ -197,7 +221,7 @@ export class PushService {
         if (options.verbose) {
             callbacks?.onLog?.(`Original working directory: ${configManager.getOriginalCwd()}`);
             callbacks?.onLog?.(`Config directory (with dir option): ${configDir}`);
-            callbacks?.onLog?.(`Config file path: ${path.join(configDir, '.mj-sync.json')}`);
+            callbacks?.onLog?.(`Config file path: ${path.join(syncRootDir, '.mj-sync.json')}`);
             callbacks?.onLog?.(`Full sync config loaded: ${JSON.stringify(this.syncConfig, null, 2)}`);
             callbacks?.onLog?.(`SQL logging config: ${JSON.stringify(this.syncConfig?.sqlLogging)}`);
         }
@@ -210,13 +234,12 @@ export class PushService {
         // Setup SQL logging session with the provider if enabled
         let sqlLoggingSession = null;
         try {
-            sqlLoggingSession = await this.startSqlLogging(sqlLogger, options, configDir, callbacks);
-            // Find entity directories to process
-            // Note: If options.dir is specified, configDir already points to that directory
-            // So we don't need to pass it as specificDir
-            const entityDirs = findEntityDirectories(configDir, undefined, this.syncConfig?.directoryOrder, this.syncConfig?.ignoreDirectories, options.include, options.exclude);
-            if (entityDirs.length === 0) {
-                throw new Error('No entity directories found');
+            sqlLoggingSession = await this.startSqlLogging(sqlLogger, options, syncRootDir, callbacks);
+            const selection = folders.SelectDirectories();
+            const entityDirs = selection.Included;
+            this.skippedDirectories = selection.Skipped;
+            for (const skipped of selection.Skipped) {
+                callbacks?.onLog?.(SkippedDirectoryMessage(configDir, skipped));
             }
             await this.applyWritePlan(entityDirs, options, callbacks);
             await this.preloadMetadata(entityDirs, options, callbacks);
@@ -253,7 +276,8 @@ export class PushService {
                         deferred: 0,
                         errors: 0,
                         warnings: this.warnings,
-                        changeLog: this.changeDetails
+                        changeLog: this.changeDetails,
+                        SkippedDirectories: this.skippedDirectories
                     };
                 }
             }
@@ -263,7 +287,7 @@ export class PushService {
             if (!options.dryRun) {
                 await transactionManager.beginTransaction();
             }
-            const totals = await this.runPushInTransaction({ entityDirs, deletionAudit, options, fileBackupManager, callbacks, configDir }, transactionManager);
+            const totals = await this.runPushInTransaction({ entityDirs, deletionAudit, options, fileBackupManager, callbacks, syncRootDir, belowSyncRoot: syncRootDir !== configDir }, transactionManager);
             if (!options.dryRun) {
                 await fileBackupManager.cleanup();
                 if (options.verbose) {
@@ -281,16 +305,19 @@ export class PushService {
                 await sqlLoggingSession.dispose();
                 if (hadStatements) {
                     sqlLogPath = filePath;
+                    this.addEnvSqlWarnings(filePath, callbacks);
                     if (options.verbose) {
                         callbacks?.onLog?.(`📝 SQL log written to: ${filePath}`);
                     }
                 }
+                this.closeEnvSqlRedactor();
             }
             return {
                 ...totals,
                 warnings: this.warnings,
                 sqlLogPath,
-                changeLog: this.changeDetails
+                changeLog: this.changeDetails,
+                SkippedDirectories: this.skippedDirectories
             };
         }
         catch (error) {
@@ -308,10 +335,16 @@ export class PushService {
             if (sqlLoggingSession) {
                 try {
                     await sqlLoggingSession.dispose();
+                    if (await fs.pathExists(sqlLoggingSession.filePath)) {
+                        this.addEnvSqlWarnings(sqlLoggingSession.filePath, callbacks);
+                    }
                 }
                 catch (disposeError) {
                     callbacks?.onWarn?.(`Failed to close SQL logging session: ${disposeError}`);
                 }
+                finally {
+                    this.closeEnvSqlRedactor();
+                }
             }
             throw error;
         }
@@ -321,7 +354,7 @@ export class PushService {
         return this.Push(options, callbacks);
     }
     /** Open the SQL logging session when the config asks for one and this is not a dry run. */
-    async startSqlLogging(sqlLogger, options, configDir, callbacks) {
+    async startSqlLogging(sqlLogger, options, syncRootDir, callbacks) {
         let sqlLoggingSession = null;
         if (sqlLogger.enabled && !options.dryRun) {
             const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
@@ -336,15 +369,24 @@ export class PushService {
                 const filename = this.syncConfig?.sqlLogging?.formatAsMigration
                     ? `MetadataSync_Push_${timestamp}.sql`
                     : `push_${timestamp}.sql`;
-                // Use .sql-log-push directory in the config directory (where sync was initiated)
-                const outputDir = path.join(configDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
+                // The SQL log folder is relative to the sync root
+                const outputDir = path.join(syncRootDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
                 const filepath = path.join(outputDir, filename);
                 // Ensure the directory exists
                 await fs.ensureDir(path.dirname(filepath));
                 // Create the SQL logging session
                 this.sqlLogFilePath = filepath;
+                this.envSqlLog = new EnvSqlLog(this.syncEngine);
+                this.lookupSqlLog = new LookupSqlLog(this.syncEngine);
+                const schemaPlaceholders = ResolveSchemaPlaceholders(this.syncConfig, configManager.GetMJConfig());
                 sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
+                    // Env values are left out first, then lookup IDs become subqueries; the logger's schema
+                    // placeholder rewrite runs after both, so the subqueries' schema names become placeholders.
+                    TransformStatement: ComposeStatementTransforms(this.envSqlLog.Redactor.Transform, this.lookupSqlLog.Rewriter.Transform),
                     formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
+                    schemaPlaceholders,
+                    // Escaping is tied to migration output, never to whether placeholder rules exist
+                    escapeFlywaySyntax: this.syncConfig?.sqlLogging?.formatAsMigration === true || this.syncConfig?.sqlLogging?.escapeFlywaySyntax === true,
                     description: 'MetadataSync push operation',
                     statementTypes: "mutations",
                     prettyPrint: true,
@@ -353,6 +395,7 @@ export class PushService {
                     filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
                     filterType: this.syncConfig?.sqlLogging?.filterType,
                     verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
+                    variableBatchThreshold: this.syncConfig?.sqlLogging?.variableBatchThreshold ?? 200,
                 });
                 if (options.verbose) {
                     callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
@@ -419,8 +462,18 @@ export class PushService {
             catch (cleanupError) {
                 callbacks?.onWarn?.(`Failed to clean up SQL logging session: ${cleanupError}`);
             }
+            finally {
+                this.closeEnvSqlRedactor();
+            }
         }
     }
+    /** Drops the SQL log redactor and this push's lookup-create hook, after the log is closed. */
+    closeEnvSqlRedactor() {
+        this.envSqlLog?.Close();
+        this.envSqlLog = null;
+        this.lookupSqlLog?.Close();
+        this.lookupSqlLog = null;
+    }
     /**
      * Decide, once and up front, how each entity directory writes: shared by default, isolated where
      * the entity (or the root, or the CLI flag) asks for it. Deciding it here means a push can never
@@ -526,7 +579,7 @@ export class PushService {
                 committed = true;
                 // PHASE 3: files that contained deletions, then the incremental state
                 await this.writeDeferredFiles(run.options, run.callbacks);
-                await this.persistIncrementalState(run.configDir);
+                await this.persistIncrementalState(run.syncRootDir, run.belowSyncRoot);
             }
             return totals;
         }
@@ -653,7 +706,7 @@ export class PushService {
             : '❌ The push transaction was still open when the push ended, and rolling it back failed.');
     }
     /** Apply incremental checksums and push timestamps. Called only after the push commits. */
-    async persistIncrementalState(syncRootDir) {
+    async persistIncrementalState(syncRootDir, belowSyncRoot) {
         if (!this.stateManager) {
             return;
         }
@@ -664,7 +717,7 @@ export class PushService {
         for (const relativeEntityDir of this.pushedEntityDirs) {
             this.stateManager.setLastPushTimestamp(relativeEntityDir, pushedAt);
         }
-        await this.stateManager.pruneStaleChecksums(syncRootDir);
+        await this.stateManager.pruneStaleChecksums(syncRootDir, belowSyncRoot ? this.pushedEntityDirs : undefined);
         await this.stateManager.save();
     }
     /** PHASE 1 over every entity directory, in order. */
@@ -686,7 +739,7 @@ export class PushService {
             this.useDirectoryMode(entityDir, options);
             this.announceDirectory(dirName, progressPrefix, entityConfig.entity, entityDir, options, callbacks);
             const firstChange = this.changeDetails.length;
-            const result = await this.processEntityDirectory(entityDir, entityConfig, options, run.fileBackupManager, callbacks, run.configDir);
+            const result = await this.processEntityDirectory(entityDir, entityConfig, options, run.fileBackupManager, callbacks, run.syncRootDir);
             // Nested relatedEntities land in changeDetails under their own entity name, so the
             // directory's configured entity is not what decides the reload (MJ#4836).
             await this.reloadMetadataIfTouched(firstChange, options.dryRun);
@@ -758,14 +811,8 @@ export class PushService {
             callbacks?.onWarn?.(`⚠️  ${subclassWarning}`);
         }
         // Find all JSON files in the directory
-        const pattern = entityConfig.filePattern || '*.json';
-        const files = await fastGlob(pattern, {
-            cwd: entityDir,
-            absolute: true,
-            onlyFiles: true,
-            dot: true,
-            ignore: ['**/node_modules/**', '**/.mj-*.json']
-        });
+        const pattern = entityConfig.filePattern || DEFAULT_RECORD_FILE_PATTERN;
+        const files = await fastGlob(pattern, { ...RECORD_FILE_GLOB_OPTIONS, cwd: entityDir });
         if (options.verbose) {
             callbacks?.onLog?.(`Found ${files.length} files to process`);
         }
@@ -799,12 +846,13 @@ export class PushService {
                 // Keep unprocessed data to write back (preserves @file: references)
                 const unprocessedRecords = Array.isArray(rawFileData) ? rawFileData : [rawFileData];
                 const isArray = Array.isArray(rawFileData);
-                // Compute checksum on the RESOLVED content (after @include preprocessing)
-                // so that changes to included files are properly detected.
+                // Compute checksum on the RESOLVED content (after @include preprocessing), with the content
+                // of each @file: target folded in, so a change to an included or referenced file is
+                // detected (#4190). @file: paths resolve against the entity folder, as push resolves them.
                 let cachedChecksum;
                 if (options.incremental && this.stateManager && syncRootDir) {
                     const relativePath = path.relative(syncRootDir, filePath);
-                    cachedChecksum = this.syncEngine.calculateChecksum(fileData);
+                    cachedChecksum = await this.syncEngine.CalculateChecksumWithFileContent(fileData, entityDir);
                     if (!this.stateManager.hasFileChanged(relativePath, cachedChecksum)) {
                         const recordCount = Array.isArray(rawFileData) ? rawFileData.length : 1;
                         if (options.verbose) {
@@ -818,6 +866,9 @@ export class PushService {
                 // Analyze dependencies and get sorted records
                 const analyzer = new RecordDependencyAnalyzer();
                 const analysisResult = await analyzer.analyzeFileRecords(records, entityConfig.entity);
+                for (const flattened of analysisResult.sortedRecords) {
+                    flattened.FilePath = filePath;
+                }
                 if (analysisResult.circularDependencies.length > 0) {
                     callbacks?.onWarn?.(`⚠️  Circular dependencies detected in ${filePath}`);
                     for (const cycle of analysisResult.circularDependencies) {
@@ -913,7 +964,7 @@ export class PushService {
                 // Uses resolved content (after @include) so included-file changes are tracked.
                 if (this.stateManager && syncRootDir) {
                     const relativePath = path.relative(syncRootDir, filePath);
-                    const checksum = cachedChecksum ?? this.syncEngine.calculateChecksum(fileData);
+                    const checksum = cachedChecksum ?? (await this.syncEngine.CalculateChecksumWithFileContent(fileData, entityDir));
                     this.pendingChecksums.set(relativePath, checksum);
                 }
             }
@@ -1056,8 +1107,8 @@ export class PushService {
         pendingWrites.set(outcome.graphId, writes);
     }
     throwRecordFailure(failure, callbacks) {
-        const err = failure.error instanceof Error ? failure.error : new Error(String(failure.error));
         const rec = failure.record;
+        const err = this.locateRecordError(failure.error instanceof Error ? failure.error : new Error(String(failure.error)), rec);
         callbacks?.onLog?.(`\n❌ Processing failed for ${rec.entityName} at ${rec.path}`);
         callbacks?.onLog?.(`   ${err.message}\n`);
         if (err.stack) {
@@ -1065,6 +1116,33 @@ export class PushService {
         }
         throw err;
     }
+    /**
+     * For an unset `@env:` variable, an error that names the record file with the record's start line,
+     * the entity and the field: `/abs/file.json:12: Entity fields.Name uses @env:VAR, which is not set`.
+     * Other errors are returned as they are.
+     */
+    locateRecordError(error, record) {
+        if (!(error instanceof EnvVariableNotSetError)) {
+            return error;
+        }
+        const field = error.Field ?? 'a field';
+        const message = `${this.recordLocation(record)}: ${error.EntityName ?? record.entityName} ${field} uses @env:${error.VariableName}, which is not set`;
+        return new Error(message, { cause: error });
+    }
+    /** `/abs/file.json:12` for a record from a file (the line where the record starts, when found), else the record path. */
+    recordLocation(record) {
+        if (!record.FilePath) {
+            return record.path;
+        }
+        const filePath = path.resolve(record.FilePath);
+        try {
+            const line = RecordStartLine(fs.readFileSync(filePath, 'utf8'), record.path);
+            return line === undefined ? filePath : `${filePath}:${line}`;
+        }
+        catch {
+            return filePath;
+        }
+    }
     async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, allowDefer = true, recordProvider) {
         const metadata = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
         const { record, entityName, parentContext, id: recordId } = flattenedRecord;
@@ -1080,13 +1158,21 @@ export class PushService {
         // Use the unique record ID from the flattened record for batch context
         // This ensures we can properly find parent entities even when they're new
         const lookupKey = recordId;
-        // Check if already in batch context
+        // Check if already in batch context; the deferred retry runs again for a record the main pass added
         const existingEntry = batchContext.get(lookupKey);
-        if (existingEntry) {
+        if (existingEntry && allowDefer) {
             // Already processed
             return { status: 'unchanged', isDuplicate: true };
         }
         let entity;
+        // Parent entity from the batch context, for @parent refs in primaryKey
+        let parentEntity = null;
+        if (parentContext) {
+            const parentRecordId = flattenedRecord.dependencies.values().next().value;
+            if (parentRecordId) {
+                parentEntity = batchContext.get(parentRecordId) || null;
+            }
+        }
         // Record-level skip (--incremental only): if the record has a stored
         // checksum and it matches the current fields, nothing changed — skip
         // without touching the DB. The checksum is embedded in the JSON
@@ -1100,16 +1186,20 @@ export class PushService {
         // drift-correction behavior single-env workflows rely on, and prevents
         // silent no-ops when seeding a fresh DB from JSON files that already
         // carry stored checksums from another environment.
-        if (options.incremental && record.sync?.checksum && record.fields && record.primaryKey && !record.deleteRecord) {
+        // The deferred retry (allowDefer false) never takes this shortcut: the first pass left fields unsaved
+        if (allowDefer && options.incremental && record.sync?.checksum && record.fields && record.primaryKey && !record.deleteRecord) {
             // Use calculateChecksumWithFileContent so @file: reference changes are detected.
             // This reads local files (fast) rather than hitting the DB (slow). The stored
             // checksum was also computed with file content and composition payloads, so the comparison is apples-to-apples.
             const checksumPayload = this.buildRecordChecksumPayload(record);
             const currentChecksum = await this.syncEngine.calculateChecksumWithFileContent(checksumPayload, entityDir);
             if (currentChecksum === record.sync.checksum) {
-                // Lightweight stub — just enough for @parent:ID and @lookup resolution.
-                // No DB call, no entity framework overhead.
-                const allFields = { ...record.primaryKey, ...record.fields };
+                // Lightweight stub — just enough for @parent:ID and @lookup resolution. No entity load.
+                // @env:, @parent: and @owner: key values are resolved without a DB call, so a child's @parent:
+                // reference copies the value; other key values (such as @lookup:) stay raw.
+                const keyCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map() };
+                const shortcutKey = await this.resolvePrimaryKey(record.primaryKey, entityDir, parentEntity, batchContext, keyCollector, recordProvider, true);
+                const allFields = { ...record.fields, ...shortcutKey };
                 const stub = {
                     EntityInfo: { Name: entityName, PrimaryKeys: Object.keys(record.primaryKey).map(k => ({ Name: k })), Fields: Object.keys(allFields).map(k => ({ Name: k })) },
                     Get: (field) => allFields[field],
@@ -1126,37 +1216,18 @@ export class PushService {
         if (!entity) {
             throw new Error(`Failed to create entity object for ${entityName}`);
         }
-        // Get parent entity from context if available (needed for @parent refs in primaryKey)
-        let parentEntity = null;
-        if (parentContext) {
-            const parentRecordId = flattenedRecord.dependencies.values().next().value;
-            if (parentRecordId) {
-                parentEntity = batchContext.get(parentRecordId) || null;
-            }
-            if (!parentEntity) {
-                throw new Error(`Parent entity not found in batch context for ${entityName}. Parent dependencies: ${Array.from(flattenedRecord.dependencies).join(', ')}`);
-            }
+        if (parentContext && !parentEntity) {
+            throw new Error(`Parent entity not found in batch context for ${entityName}. Parent dependencies: ${Array.from(flattenedRecord.dependencies).join(', ')}`);
         }
         // Check if record exists
         // Process primaryKey values through processFieldValue to resolve @lookup, @parent, etc.
         // Create a resolution collector to track @lookup and @parent resolutions
-        const resolutionCollector = { notes: [], fieldPrefix: 'primaryKey' };
+        const resolutionCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map(), LookupSources: new Map() };
+        // Reads the same EnvFields map, so fields recorded later are masked too
+        const envMask = new EnvDisplayMask(resolutionCollector.EnvFields);
         let resolvedPrimaryKey;
         if (record.primaryKey && Object.keys(record.primaryKey).length > 0) {
-            resolvedPrimaryKey = {};
-            for (const [pkField, pkValue] of Object.entries(record.primaryKey)) {
-                try {
-                    resolvedPrimaryKey[pkField] = await this.syncEngine.processFieldValue(pkValue, entityDir, parentEntity, null, // rootRecord
-                    0, batchContext, resolutionCollector, pkField, recordProvider);
-                }
-                catch (pkError) {
-                    // Check if this is a deferrable lookup error
-                    if (pkError instanceof DeferrableLookupError) {
-                        throw new Error(`Cannot defer lookup in primaryKey field '${pkField}': ${pkError.message}. Primary key lookups must resolve immediately.`);
-                    }
-                    throw pkError;
-                }
-            }
+            resolvedPrimaryKey = await this.resolvePrimaryKey(record.primaryKey, entityDir, parentEntity, batchContext, resolutionCollector, recordProvider);
         }
         let exists = false;
         let isNew = false;
@@ -1176,7 +1247,7 @@ export class PushService {
                 // Record doesn't exist in database
                 const autoCreate = this.syncConfig?.push?.autoCreateMissingRecords ?? false;
                 const pkDisplay = Object.entries(resolvedPrimaryKey)
-                    .map(([key, value]) => `${key}=${value}`)
+                    .map(([key, value]) => `${key}=${envMask.NewValue(key, String(value))}`)
                     .join(', ');
                 if (!autoCreate) {
                     const warning = `Record not found: ${entityName} with primaryKey {${pkDisplay}}. To auto-create missing records, set push.autoCreateMissingRecords=true in .mj-sync.json`;
@@ -1212,11 +1283,32 @@ export class PushService {
         }
         // Apply defaults if entityConfig is provided
         if (entityConfig) {
-            const defaults = await this.syncEngine.buildDefaults(flattenedRecord.path, entityConfig);
+            const defaultsCollector = { notes: [], fieldPrefix: 'fields', EnvFields: new Map() };
+            // Folder defaults come from the record file's folders
+            const recordFile = flattenedRecord.FilePath ?? path.join(entityDir, '.record.json');
+            const skippedDefaults = new Map();
+            const defaults = await this.syncEngine.buildDefaults(recordFile, entityConfig, defaultsCollector, skippedDefaults);
+            // Field names match the entity's fields without case; a default for a virtual property applies when the entity has that property
+            const recordFields = new RecordFieldNames(record.fields);
+            // A default left out for an unset optional variable warns when the record would have used it
+            for (const [field, variableName] of skippedDefaults) {
+                const name = EntityFieldName(entity, field) ?? (field in entity ? field : undefined);
+                if (name && !recordFields.Has(name)) {
+                    const warning = OptionalEnvSkipMessage(entityName, name, variableName);
+                    localWarnings.push(warning);
+                    callbacks?.onWarn?.(`⚠️  ${warning}`);
+                    AddSkippedOptionalEnvField(resolutionCollector.EnvFields, name, variableName);
+                }
+            }
             // Apply defaults only to fields not explicitly set in record.fields
             for (const [field, value] of Object.entries(defaults)) {
-                if (!(field in record.fields) && field in entity) {
-                    entity.Set(field, value);
+                const name = EntityFieldName(entity, field) ?? (field in entity ? field : undefined);
+                if (name && !recordFields.Has(name)) {
+                    entity.Set(name, value);
+                    const variables = defaultsCollector.EnvFields?.get(field);
+                    if (variables) {
+                        resolutionCollector.EnvFields?.set(name, variables);
+                    }
                 }
             }
         }
@@ -1277,6 +1369,16 @@ export class PushService {
                 }
             }
             catch (fieldError) {
+                if (fieldError instanceof EnvVariableNotSetError) {
+                    throw fieldError.ForField(`fields.${fieldName}`);
+                }
+                if (fieldError instanceof OptionalEnvMissingError) {
+                    const warning = OptionalEnvSkipMessage(entityName, fieldName, fieldError.VariableName);
+                    localWarnings.push(warning);
+                    callbacks?.onWarn?.(`⚠️  ${warning}`);
+                    AddSkippedOptionalEnvField(resolutionCollector.EnvFields, EntityFieldName(entity, fieldName) ?? fieldName, fieldError.VariableName);
+                    continue;
+                }
                 // Check if this is a deferrable lookup error first
                 if (fieldError instanceof DeferrableLookupError) {
                     // If allowDefer is false, we're in deferred processing mode - can't defer again
@@ -1376,15 +1478,16 @@ export class PushService {
                 const primaryKeyDisplay = [];
                 if (entityInfo) {
                     for (const pk of entityInfo.PrimaryKeys) {
-                        primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                        primaryKeyDisplay.push(`${pk.Name}: ${envMask.NewValue(pk.Name, String(entity.Get(pk.Name)))}`);
                     }
                 }
                 const fieldDiffs = changeKeys.map((fieldName) => {
                     const field = entity.GetFieldByName(fieldName);
+                    const envName = field?.Name ?? fieldName;
                     return {
                         field: fieldName,
-                        oldValue: this.formatFieldValue(field ? field.OldValue : undefined),
-                        newValue: this.formatFieldValue(changes[fieldName]),
+                        oldValue: envMask.OldValue(envName, this.formatFieldValue(field ? field.OldValue : undefined)),
+                        newValue: envMask.NewValue(envName, this.formatFieldValue(changes[fieldName])),
                     };
                 });
                 this.changeDetails.push({
@@ -1415,6 +1518,11 @@ export class PushService {
                 status: 'unchanged',
             };
         }
+        // A literal key that missed can still collide on a unique key with a row this database made
+        // under another ID: report that row instead of a constraint violation (#4400).
+        if (isNew && resolvedPrimaryKey && Object.keys(resolvedPrimaryKey).length > 0) {
+            await this.checkNaturalKey(entity, resolvedPrimaryKey, envMask, recordProvider);
+        }
         // A dry run stops here, after the same change detection a real push uses, so an
         // in-sync record reports unchanged above rather than "would update" (#4529).
         if (options.dryRun) {
@@ -1424,7 +1532,7 @@ export class PushService {
             const batchContextEntry = { key: lookupKey, entity };
             if (exists) {
                 callbacks?.onLog?.(`[DRY RUN] Would update ${entityName} record`);
-                return { status: 'updated', batchContextEntry };
+                return { status: 'updated', batchContextEntry, warnings: localWarnings.length > 0 ? localWarnings : undefined };
             }
             else {
                 callbacks?.onLog?.(`[DRY RUN] Would create ${entityName} record`);
@@ -1433,20 +1541,27 @@ export class PushService {
                 // a throwaway UUID that changes every run and is never the one a push writes.
                 const entityInfo = this.syncEngine.getEntityInfo(entityName);
                 const primaryKeyDisplay = (entityInfo?.PrimaryKeys ?? [])
-                    .filter((pk) => resolvedPrimaryKey?.[pk.Name] != null || record.fields?.[pk.Name] != null)
-                    .map((pk) => `${pk.Name}: ${resolvedPrimaryKey?.[pk.Name] ?? entity.Get(pk.Name)}`);
+                    .filter((pk) => KeyFieldValue(resolvedPrimaryKey, pk.Name) != null || KeyFieldValue(record.fields, pk.Name) != null)
+                    .map((pk) => `${pk.Name}: ${envMask.NewValue(pk.Name, String(KeyFieldValue(resolvedPrimaryKey, pk.Name) ?? entity.Get(pk.Name)))}`);
                 this.changeDetails.push({
                     entityName,
                     primaryKey: primaryKeyDisplay.length > 0 ? primaryKeyDisplay.join(', ') : 'new',
                     Operation: 'created',
                     fields: [],
                 });
-                return { status: 'created', batchContextEntry };
+                return { status: 'created', batchContextEntry, warnings: localWarnings.length > 0 ? localWarnings : undefined };
             }
         }
+        this.registerEnvSqlFields(entity, resolutionCollector.EnvFields, isNew || alwaysPush);
+        this.registerPendingEnvSqlChildFields(resolutionCollector);
+        // Key and foreign-key values from @lookup are logged as subqueries on the natural key (#5626)
+        this.lookupSqlLog?.Rewriter.RegisterEntityLookups(entity, resolutionCollector.LookupSources);
         // Save the record with detailed error logging
+        const keyBeforeSave = new Map(entity.PrimaryKeys.map((pk) => [pk.Name, entity.Get(pk.Name)]));
         const recordName = entity.Get('Name');
-        const entityRecordId = entity.Get('ID');
+        const recordKeyText = this.recordIdLabel(entity, envMask);
+        // Free error text can hold a value of any variable push resolved, composition children included
+        const errorTextMask = EnvDisplayMask.ForVariables(this.syncEngine.ResolvedEnvVariables);
         let saveResult;
         try {
             // Skip embedding generation during sync — vectors can be computed later by the
@@ -1466,10 +1581,10 @@ export class PushService {
                 callbacks?.onLog?.(msg);
             };
             logError(`\n❌ SAVE EXCEPTION for ${entityName}`);
-            logError(`   Record ID: ${entityRecordId || 'NEW'}`);
-            logError(`   Record Name: ${recordName || 'N/A'}`);
+            logError(`   Record ID: ${recordKeyText}`);
+            logError(`   Record Name: ${recordName ? envMask.NewValue('Name', String(recordName)) : 'N/A'}`);
             logError(`   File Path: ${flattenedRecord.path}`);
-            logError(`   Error: ${saveError.message || saveError}`);
+            logError(`   Error: ${errorTextMask.MaskText(String(saveError.message || saveError))}`);
             // Check for specific error patterns
             if (saveError.message?.includes('Cannot insert the value NULL')) {
                 logError(`   Tip: A required field is NULL. Check the entity's required fields.`);
@@ -1491,11 +1606,11 @@ export class PushService {
                     const displayValue = typeof value === 'string' && value.length > 100
                         ? value.substring(0, 100) + '...'
                         : value;
-                    logError(`     ${field.Name}: ${displayValue}`);
+                    logError(`     ${field.Name}: ${envMask.NewValue(field.Name, String(displayValue))}`);
                 }
             }
             logError(''); // Empty line for readability
-            throw saveError;
+            throw MaskedError(saveError, errorTextMask);
         }
         if (!saveResult) {
             // Helper to log to both console and callbacks
@@ -1504,14 +1619,14 @@ export class PushService {
                 callbacks?.onLog?.(msg);
             };
             logError(`\n❌ SAVE RETURNED FALSE for ${entityName}`);
-            logError(`   Record ID: ${entityRecordId || 'NEW'}`);
-            logError(`   Record Name: ${recordName || 'N/A'}`);
+            logError(`   Record ID: ${recordKeyText}`);
+            logError(`   Record Name: ${recordName ? envMask.NewValue('Name', String(recordName)) : 'N/A'}`);
             logError(`   File Path: ${flattenedRecord.path}`);
             // Log the LatestResult for debugging
             if (entity.LatestResult) {
                 const completeMessage = entity.LatestResult.CompleteMessage;
                 if (completeMessage) {
-                    logError(`   Database Error: ${completeMessage}`);
+                    logError(`   Database Error: ${errorTextMask.MaskText(completeMessage)}`);
                 }
                 if (entity.LatestResult.SQL) {
                     // Don't log the full SQL as it might be huge, just indicate it's available
@@ -1526,7 +1641,7 @@ export class PushService {
                     const displayValue = typeof value === 'string' && value.length > 100
                         ? value.substring(0, 100) + '...'
                         : value;
-                    logError(`     ${field.Name}: ${displayValue}`);
+                    logError(`     ${field.Name}: ${envMask.NewValue(field.Name, String(displayValue))}`);
                 }
             }
             logError(''); // Empty line for readability
@@ -1538,17 +1653,18 @@ export class PushService {
             if (entityInfo) {
                 for (const pk of entityInfo.PrimaryKeys) {
                     const pkValue = entity.Get(pk.Name);
-                    primaryKeyInfo.push(`${pk.Name}=${this.formatFieldValue(pkValue)}`);
+                    primaryKeyInfo.push(`${pk.Name}=${envMask.NewValue(pk.Name, this.formatFieldValue(pkValue))}`);
                 }
             }
             // Collect field values that were being saved
             for (const [fieldName, fieldValue] of Object.entries(record.fields)) {
                 const processedValue = entity.Get(fieldName);
-                fieldInfo.push(`${fieldName}=${this.formatFieldValue(processedValue)}`);
+                const envName = entity.GetFieldByName(fieldName)?.Name ?? fieldName;
+                fieldInfo.push(`${fieldName}=${envMask.NewValue(envName, this.formatFieldValue(processedValue))}`);
             }
             // Get the actual error details from the entity
-            const errorMessage = entity.LatestResult?.CompleteMessage || 'Unknown error';
-            const errorDetails = entity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || '';
+            const errorMessage = errorTextMask.MaskText(entity.LatestResult?.CompleteMessage || 'Unknown error');
+            const errorDetails = errorTextMask.MaskText(entity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || '');
             // Log detailed error information
             callbacks?.onError?.(`\n❌ FATAL ERROR: Failed to save ${entityName} record`);
             callbacks?.onError?.(`   Entity: ${entityName}`);
@@ -1597,7 +1713,8 @@ export class PushService {
             deferredRecord = {
                 flattenedRecord,
                 entityDir,
-                entityConfig
+                entityConfig,
+                batchContext
             };
             if (options.verbose) {
                 callbacks?.onLog?.(`   📋 Queued ${entityName} for deferred processing (record saved, some fields pending)`);
@@ -1611,9 +1728,10 @@ export class PushService {
             if (entityInfo) {
                 const newPrimaryKey = {};
                 const primaryKeyDisplay = [];
-                for (const pk of entityInfo.PrimaryKeys) {
-                    newPrimaryKey[pk.Name] = entity.Get(pk.Name);
-                    primaryKeyDisplay.push(`${pk.Name}: ${entity.Get(pk.Name)}`);
+                // An env-sourced key keeps the file's reference unless the database assigned another value
+                for (const key of CreatedRecordKeyFields(entityInfo.PrimaryKeys, entity, keyBeforeSave, record.primaryKey, originalFields, envMask)) {
+                    newPrimaryKey[key.Name] = key.FileValue;
+                    primaryKeyDisplay.push(`${key.Name}: ${key.KeptBySave ? envMask.NewValue(key.Name, String(key.SavedValue)) : String(key.SavedValue)}`);
                 }
                 record.primaryKey = newPrimaryKey;
                 // Capture the create for the changes recap (no field diffs — the whole record is new).
@@ -1631,6 +1749,12 @@ export class PushService {
         if (!shouldWriteSync) {
             delete record.sync;
         }
+        else if (deferredRecord) {
+            // A deferred record keeps its earlier checksum in the file, so each later --incremental push checks it again until the record changes
+            if (options.verbose) {
+                callbacks?.onLog?.(`   - Sync metadata waits for the deferred fields`);
+            }
+        }
         else if (isNew || isDirty) {
             const checksumPayload = this.buildRecordChecksumPayload(record, originalFields);
             const checksum = await this.syncEngine.calculateChecksumWithFileContent(checksumPayload, entityDir);
@@ -1695,6 +1819,77 @@ export class PushService {
             status: isNew ? 'created' : (isDirty ? 'updated' : 'unchanged'),
         };
     }
+    /**
+     * Resolves each primary-key value (`@lookup`, `@parent`, `@env`, …). A key lookup cannot be deferred.
+     * With `referencesOnly`, only `@env:`, `@parent:` and `@owner:` values are resolved; others are copied as is.
+     */
+    async resolvePrimaryKey(primaryKey, entityDir, parentEntity, batchContext, resolutionCollector, recordProvider, referencesOnly = false) {
+        const resolved = {};
+        for (const [pkField, pkValue] of Object.entries(primaryKey)) {
+            if (referencesOnly && !IsKeyReference(pkValue)) {
+                resolved[pkField] = pkValue;
+                continue;
+            }
+            try {
+                resolved[pkField] = await this.syncEngine.processFieldValue(pkValue, entityDir, parentEntity, null, // rootRecord
+                0, batchContext, resolutionCollector, pkField, recordProvider);
+            }
+            catch (pkError) {
+                if (pkError instanceof EnvVariableNotSetError) {
+                    throw pkError.ForField(`primaryKey.${pkField}`);
+                }
+                if (pkError instanceof DeferrableLookupError) {
+                    throw new Error(`Cannot defer lookup in primaryKey field '${pkField}': ${pkError.message}. Primary key lookups must resolve immediately.`);
+                }
+                throw pkError;
+            }
+        }
+        return resolved;
+    }
+    /** A file key value for printing: `<from @env:VAR>` for an `@env:` reference, else the value. */
+    fileKeyText(value) {
+        const reference = ParseEnvReference(value);
+        return reference ? EnvSourceLabel([reference.VariableName]) : String(value);
+    }
+    /**
+     * The key with each `@env:` value resolved through `collector`, which records the env-sourced fields.
+     * Other values are copied as written. A malformed or unset reference throws a names-only error.
+     */
+    async resolveEnvKeyValues(primaryKey, entityDir, collector) {
+        const resolved = {};
+        for (const [field, value] of Object.entries(primaryKey)) {
+            resolved[field] = typeof value === 'string' && value.startsWith(METADATA_KEYWORDS.ENV)
+                ? await this.syncEngine.processFieldValue(value, entityDir, null, null, 0, undefined, collector, field)
+                : value;
+        }
+        return resolved;
+    }
+    /** True when the child holds every key value; UUID key fields match without regard to case. */
+    childHasKey(child, key) {
+        return Object.entries(key).every(([field, value]) => {
+            const current = child.Get(field);
+            const isUuidField = child.GetFieldByName(field)?.EntityFieldInfo?.IsUniqueIdentifier === true;
+            if (isUuidField && typeof current === 'string' && typeof value === 'string') {
+                return UUIDsEqual(current, value);
+            }
+            return String(current) === String(value);
+        });
+    }
+    /**
+     * The record's key for save-failure output: the value of a single key field, or `Field=value` pairs
+     * for a composite key, with env-sourced key fields shown as labels. `NEW` when no key field has a value.
+     */
+    recordIdLabel(entity, envMask) {
+        const keyFields = entity.PrimaryKeys.filter((pk) => {
+            const value = entity.Get(pk.Name);
+            return value !== null && value !== undefined && value !== '';
+        });
+        if (keyFields.length === 0) {
+            return 'NEW';
+        }
+        const labels = keyFields.map((pk) => envMask.NewValue(pk.Name, String(entity.Get(pk.Name))));
+        return keyFields.length === 1 ? labels[0] : keyFields.map((pk, i) => `${pk.Name}=${labels[i]}`).join(', ');
+    }
     formatFieldValue(value) {
         if (value === null || value === undefined)
             return 'null';
@@ -1711,14 +1906,19 @@ export class PushService {
         }
         return String(value);
     }
-    async processDeleteRecord(flattenedRecord, _entityDir, options, callbacks, isDbOnly = false) {
+    async processDeleteRecord(flattenedRecord, entityDir, options, callbacks, isDbOnly = false) {
         const { record, entityName } = flattenedRecord;
         // Validate that we have a primary key for deletion
         if (!record.primaryKey || Object.keys(record.primaryKey).length === 0) {
             throw new Error(`Cannot delete ${entityName} record without primaryKey. Please specify primaryKey fields.`);
         }
-        // Load the entity to check if it exists in the target database
-        const existingEntity = await this.syncEngine.loadEntity(entityName, record.primaryKey);
+        // Load the entity to check if it exists in the target database; an `@env:` key value is resolved first
+        const keyCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map() };
+        const resolvedKey = await this.resolveEnvKeyValues(record.primaryKey, entityDir, keyCollector);
+        const keyMask = new EnvDisplayMask(keyCollector.EnvFields);
+        // Read through the push transaction's provider, not the preload cache: a row an earlier delete's
+        // cascade removed in this push is still in the cache until commit (#4935).
+        const existingEntity = await this.syncEngine.LoadEntityFromDatabase(entityName, resolvedKey, this.transactionProvider());
         // Check if the deletion has already been processed
         if (record.deleteRecord?.deletedAt) {
             // Verify if record still exists in THIS database
@@ -1736,26 +1936,16 @@ export class PushService {
             // Fall through to deletion logic
         }
         if (!existingEntity) {
-            const pkDisplay = Object.entries(record.primaryKey)
-                .map(([key, value]) => `${key}=${value}`)
-                .join(', ');
-            const warning = `Record not found for deletion: ${entityName} with primaryKey {${pkDisplay}}`;
-            this.warnings.push(warning);
-            callbacks?.onWarn?.(warning);
-            // Mark as deleted anyway since it doesn't exist
-            if (!record.deleteRecord) {
-                record.deleteRecord = { delete: true };
-            }
-            record.deleteRecord.deletedAt = undefined; // Indicate it was not found
-            record.deleteRecord.notFound = true;
-            return { status: 'skipped', isDuplicate: false };
+            return this.skipMissingDelete(flattenedRecord, 'Record not found for deletion', callbacks);
         }
+        // Key values and free error text can hold a value of any variable push resolved
+        const errorTextMask = EnvDisplayMask.ForVariables(this.syncEngine.ResolvedEnvVariables);
         // Log the deletion
         const entityInfo = this.syncEngine.getEntityInfo(entityName);
         const primaryKeyDisplay = [];
         if (entityInfo) {
             for (const pk of entityInfo.PrimaryKeys) {
-                primaryKeyDisplay.push(`${pk.Name}: ${existingEntity.Get(pk.Name)}`);
+                primaryKeyDisplay.push(`${pk.Name}: ${keyMask.NewValue(pk.Name, errorTextMask.MaskText(String(existingEntity.Get(pk.Name))))}`);
             }
         }
         // Deletion detail streams inline only in verbose mode — the recap is the default
@@ -1772,7 +1962,9 @@ export class PushService {
             }
             const recordNameVerbose = existingEntity.Get('Name');
             if (recordNameVerbose) {
-                callbacks?.onLog?.(`   Name: ${recordNameVerbose}`);
+                const fileNameKey = Object.keys(record.fields ?? {}).find((key) => key.toLowerCase() === 'name');
+                const nameFromEnv = fileNameKey !== undefined && ParseEnvReference(record.fields?.[fileNameKey]) !== null;
+                callbacks?.onLog?.(`   Name: ${nameFromEnv ? ENV_HIDDEN_VALUE : recordNameVerbose}`);
             }
         }
         // Capture the deletion for the changes recap (no field diffs for deletes).
@@ -1788,13 +1980,19 @@ export class PushService {
             }
             return { status: 'deleted', isDuplicate: false };
         }
+        // An env-sourced delete key gets the names-only SQL log warning
+        this.registerEnvSqlFields(existingEntity, keyCollector.EnvFields, false);
         // Delete the record
         try {
             const deleteResult = await existingEntity.Delete();
             if (!deleteResult) {
+                // A row already gone (removed by an earlier delete's cascade) is the intended end state
+                if (!(await this.rowStillExists(entityName, resolvedKey))) {
+                    return this.skipMissingDelete(flattenedRecord, 'Record was already gone when it was deleted (an earlier delete in this push may have cascaded to it)', callbacks);
+                }
                 // Check the LatestResult for error details
-                const errorMessage = existingEntity.LatestResult?.CompleteMessage || 'Unknown error';
-                const errorDetails = existingEntity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || '';
+                const errorMessage = errorTextMask.MaskText(existingEntity.LatestResult?.CompleteMessage || 'Unknown error');
+                const errorDetails = errorTextMask.MaskText(existingEntity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || '');
                 callbacks?.onError?.(`\n❌ Failed to delete ${entityName} record`);
                 callbacks?.onError?.(`   Primary Key: {${primaryKeyDisplay.join(', ')}}`);
                 callbacks?.onError?.(`   Error: ${errorMessage}`);
@@ -1810,7 +2008,7 @@ export class PushService {
                 throw new Error(`Failed to delete ${entityName} record: ${errorMessage}`);
             }
             // Remove from metadata cache
-            this.syncMetadataEngine.removeEntityFromCache(entityName, record.primaryKey);
+            this.syncMetadataEngine.removeEntityFromCache(entityName, resolvedKey);
             // Set deletedAt timestamp after successful deletion (only for metadata records)
             if (!isDbOnly) {
                 if (!record.deleteRecord) {
@@ -1831,9 +2029,12 @@ export class PushService {
             return { status: 'deleted', isDuplicate: false };
         }
         catch (deleteError) {
+            if (!(await this.rowStillExists(entityName, resolvedKey))) {
+                return this.skipMissingDelete(flattenedRecord, 'Record was already gone when it was deleted (an earlier delete in this push may have cascaded to it)', callbacks);
+            }
             console.error(`\n❌ DELETE EXCEPTION for ${entityName}`);
             console.error(`   Primary Key: {${primaryKeyDisplay.join(', ')}}`);
-            console.error(`   Error: ${deleteError.message || deleteError}`);
+            console.error(`   Error: ${errorTextMask.MaskText(String(deleteError.message || deleteError))}`);
             // Check for specific error patterns
             if (deleteError.message?.includes('FOREIGN KEY constraint')) {
                 console.error(`   Tip: This record is referenced by other records and cannot be deleted.`);
@@ -1842,9 +2043,64 @@ export class PushService {
             else if (deleteError.message?.includes('permission')) {
                 console.error(`   Tip: You may not have permission to delete this record.`);
             }
-            throw deleteError;
+            throw MaskedError(deleteError, errorTextMask);
+        }
+    }
+    /**
+     * A delete whose row is not in the database: a warning, the record marked not found, and a skip.
+     * The database is already in the state the delete asks for.
+     */
+    skipMissingDelete(flattenedRecord, reason, callbacks) {
+        const { record, entityName } = flattenedRecord;
+        const pkDisplay = Object.entries(record.primaryKey ?? {})
+            .map(([key, value]) => `${key}=${this.fileKeyText(value)}`)
+            .join(', ');
+        const warning = `${reason}: ${entityName} with primaryKey {${pkDisplay}}`;
+        this.warnings.push(warning);
+        callbacks?.onWarn?.(warning);
+        // Mark as deleted anyway since it doesn't exist
+        if (!record.deleteRecord) {
+            record.deleteRecord = { delete: true };
+        }
+        record.deleteRecord.deletedAt = undefined; // Indicate it was not found
+        record.deleteRecord.notFound = true;
+        return { status: 'skipped', isDuplicate: false };
+    }
+    /**
+     * True when the row is still in the database, read through the push transaction. A failed read
+     * counts as present, so the original delete error is reported.
+     */
+    async rowStillExists(entityName, primaryKey) {
+        try {
+            return (await this.syncEngine.LoadEntityFromDatabase(entityName, primaryKey, this.transactionProvider())) !== null;
+        }
+        catch {
+            return true;
         }
     }
+    /**
+     * Throws when creating `entity` with the key the record declares would collide on a unique key
+     * with an existing row, naming that row's key. See natural-key-preflight.ts.
+     */
+    async checkNaturalKey(entity, resolvedPrimaryKey, envMask, recordProvider) {
+        const host = this.hostProvider();
+        if (typeof host?.ExecuteSQL !== 'function') {
+            return; // no SQL provider, so no catalog to read unique keys from
+        }
+        // Rows are found on the record's provider, so the lookup sees what its transaction wrote
+        const reader = new DatabaseNaturalKeyReader(host, this.contextUser, recordProvider ?? this.transactionProvider(), this.uniqueKeyCache);
+        const conflict = await new NaturalKeyPreflight(reader).Check(entity);
+        if (conflict) {
+            const recordKey = Object.entries(resolvedPrimaryKey)
+                .map(([field, value]) => `${field}=${envMask.NewValue(field, String(value))}`)
+                .join(', ');
+            throw new Error(NaturalKeyConflictMessage(entity.EntityInfo.Name, recordKey, conflict));
+        }
+    }
+    /** The provider that owns the push transaction, as the metadata provider a read joins it through. */
+    transactionProvider() {
+        return this.hostProvider();
+    }
     /**
      * Prompt user for confirmation before proceeding with push operation
      * This happens after deletion audit but before any database operations
@@ -1936,7 +2192,7 @@ export class PushService {
                     }
                 }
             }
-            const pattern = entityConfig.filePattern || '*.json';
+            const pattern = entityConfig.filePattern || DEFAULT_RECORD_FILE_PATTERN;
             const files = await fastGlob(pattern, {
                 cwd: entityDir,
                 absolute: true,
@@ -1987,7 +2243,7 @@ export class PushService {
                 continue;
             }
             // Find all JSON files
-            const pattern = entityConfig.filePattern || '*.json';
+            const pattern = entityConfig.filePattern || DEFAULT_RECORD_FILE_PATTERN;
             const files = await fastGlob(pattern, {
                 cwd: entityDir,
                 absolute: true,
@@ -2010,6 +2266,9 @@ export class PushService {
                     const records = Array.isArray(fileData) ? fileData : [fileData];
                     // Phase 1: Flatten only - accumulates into analyzer's internal state
                     const flattenedRecords = analyzer.flattenFileRecords(records, entityConfig.entity);
+                    for (const flattened of flattenedRecords) {
+                        flattened.FilePath = filePath;
+                    }
                     allFlattenedRecords.push(...flattenedRecords);
                 }
                 catch (error) {
@@ -2119,9 +2378,10 @@ export class PushService {
                     }
                 }
                 catch (error) {
-                    callbacks?.onError?.(`   Failed to delete ${record.entityName}: ${error}`);
+                    const located = error instanceof Error ? this.locateRecordError(error, record) : error;
+                    callbacks?.onError?.(`   Failed to delete ${record.entityName}: ${located}`);
                     errors++;
-                    throw error; // Fail fast on first deletion error
+                    throw located; // Fail fast on first deletion error
                 }
             }
         }
@@ -2173,14 +2433,14 @@ export class PushService {
         let created = 0;
         let updated = 0;
         let errors = 0;
-        // Create a fresh batch context for deferred processing
-        // Records are in DB now, so this is mainly for tracking within this phase
-        const batchContext = new BatchContextIndex();
+        // A deferred record without the batch context of its main pass gets one for this phase
+        const phaseBatchContext = new BatchContextIndex();
         for (const deferred of this.deferredRecords) {
             const { flattenedRecord, entityDir, entityConfig } = deferred;
+            const batchContext = deferred.batchContext ?? phaseBatchContext;
             const entityName = flattenedRecord.entityName;
             const recordId = flattenedRecord.record.primaryKey
-                ? Object.entries(flattenedRecord.record.primaryKey).map(([k, v]) => `${k}=${v}`).join(', ')
+                ? Object.entries(flattenedRecord.record.primaryKey).map(([k, v]) => `${k}=${this.fileKeyText(v)}`).join(', ')
                 : (flattenedRecord.record.fields.Name || 'NEW');
             try {
                 // Re-run processFlattenedRecord with allowDefer=false
@@ -2208,7 +2468,7 @@ export class PushService {
                 }
             }
             catch (error) {
-                const err = error instanceof Error ? error : new Error(String(error));
+                const err = this.locateRecordError(error instanceof Error ? error : new Error(String(error)), flattenedRecord);
                 callbacks?.onError?.(`   ✗ Failed to process deferred record: ${entityName} (${recordId})`);
                 callbacks?.onError?.(`     Error: ${err.message}`);
                 callbacks?.onError?.(`     Tip: Ensure all referenced records exist or remove the ?allowDefer flag`);
@@ -2359,6 +2619,89 @@ export class PushService {
     buildRecordChecksumPayload(record, fieldsOverride) {
         return SyncEngine.BuildRecordChecksumPayload(fieldsOverride ?? record.fields, record.collections, record.embeds, record.extension);
     }
+    /**
+     * Resolves one field of an embed, collection item or extension and sets it on the target.
+     * A missing `@env:VAR?optional` variable leaves the field unset and raises a warning.
+     * `sharesOwnerFields` (IS-A extension children) records env-sourced fields in the owner's collector.
+     */
+    async setCompositionField(target, fieldName, rawValue, entityDir, batchContext, resolutionCollector, recordProvider, ownerRecord, callbacks, sharesOwnerFields = false) {
+        // An IS-A extension child writes owner fields, so it records into the owner's collector. Other
+        // children get their own EnvFields map, so a child field never masks an owner field of the same name.
+        const childCollector = sharesOwnerFields
+            ? resolutionCollector
+            : { notes: resolutionCollector.notes, fieldPrefix: resolutionCollector.fieldPrefix, EnvFields: new Map() };
+        try {
+            const value = await this.syncEngine.processFieldValue(rawValue, entityDir, null, null, 0, batchContext, childCollector, fieldName, recordProvider, ownerRecord);
+            target.Set(fieldName, value);
+            const variables = childCollector.EnvFields?.get(fieldName);
+            if (variables) {
+                this.queueEnvSqlChildField(resolutionCollector, { Target: target, FieldName: fieldName, Variables: variables });
+            }
+        }
+        catch (error) {
+            if (error instanceof EnvVariableNotSetError) {
+                throw error.ForField(`fields.${fieldName}`, target.EntityInfo.Name);
+            }
+            if (!(error instanceof OptionalEnvMissingError)) {
+                throw error;
+            }
+            this.addWarning(OptionalEnvSkipMessage(target.EntityInfo.Name, fieldName, error.VariableName), callbacks);
+            this.queueEnvSqlChildField(resolutionCollector, { Target: target, FieldName: fieldName, Variables: new Set([error.VariableName]) });
+        }
+    }
+    /** Runs composition work and names its path (and entity) in an unset-variable error. */
+    async withCompositionPath(compositionPath, entityName, work) {
+        try {
+            await work();
+        }
+        catch (error) {
+            if (error instanceof EnvVariableNotSetError) {
+                throw error.Within(compositionPath, entityName());
+            }
+            throw error;
+        }
+    }
+    /** Registers the env-sourced fields of this save with the SQL log redactor, when a log is open. */
+    registerEnvSqlFields(entity, envFields, fullRow) {
+        this.envSqlLog?.Redactor.RegisterEntityFields(entity, envFields, fullRow);
+    }
+    /** Keeps an env-sourced composition child field until its owner's save, when a SQL log is open. */
+    queueEnvSqlChildField(ownerCollector, field) {
+        if (!this.envSqlLog) {
+            return;
+        }
+        const pending = this.pendingEnvSqlChildFields.get(ownerCollector) ?? [];
+        pending.push(field);
+        this.pendingEnvSqlChildFields.set(ownerCollector, pending);
+    }
+    /** Queues each env-sourced key field of a composition child for the names-only SQL log warning. */
+    queueEnvSqlChildKey(ownerCollector, child, keyCollector) {
+        for (const [fieldName, variables] of keyCollector.EnvFields ?? []) {
+            this.queueEnvSqlChildField(ownerCollector, { Target: child, FieldName: fieldName, Variables: variables });
+        }
+    }
+    /** Registers the queued composition child fields of the record that is about to be saved. */
+    registerPendingEnvSqlChildFields(ownerCollector) {
+        const pending = this.pendingEnvSqlChildFields.get(ownerCollector) ?? [];
+        this.pendingEnvSqlChildFields.delete(ownerCollector);
+        for (const field of pending) {
+            this.registerEnvSqlCompositionField(field.Target, field.FieldName, field.Variables);
+        }
+    }
+    /** Registers one env-sourced composition child field when its save statement holds it. */
+    registerEnvSqlCompositionField(target, fieldName, variables) {
+        const fullRow = !target.IsSaved || this.syncConfig?.push?.alwaysPush === true;
+        this.registerEnvSqlFields(target, new Map([[fieldName, variables]]), fullRow);
+    }
+    /** Adds the names-only SQL log warnings of this run. */
+    addEnvSqlWarnings(sqlLogPath, callbacks) {
+        for (const warning of this.envSqlLog?.Redactor.Warnings(sqlLogPath) ?? []) {
+            this.addWarning(warning, callbacks);
+        }
+        for (const warning of this.lookupSqlLog?.Rewriter.Warnings() ?? []) {
+            this.addWarning(warning, callbacks);
+        }
+    }
     /**
      * Applies first-class composition axes onto the entity before Save (§5):
      * 1. embeds (peer first via {fkField}_EnsureObject() or companion.Ensure())
@@ -2405,16 +2748,17 @@ export class PushService {
                 if (!embeddedEntity) {
                     throw new Error(`Failed to resolve embedded companion for '${fkField}' on entity '${entityName}'. Ensure that DeclareEmbeddedRecord is called for '${fkField}'.`);
                 }
-                // Apply embed fields, passing ownerRecord = entity for @owner: resolution
-                if (embedRecord.fields && typeof embedRecord.fields === 'object') {
-                    for (const [fName, fVal] of Object.entries(embedRecord.fields)) {
-                        const processedValue = await this.syncEngine.processFieldValue(fVal, entityDir, null, null, 0, batchContext, resolutionCollector, fName, recordProvider, entity // ownerRecord
-                        );
-                        embeddedEntity.Set(fName, processedValue);
+                const embedded = embeddedEntity;
+                await this.withCompositionPath(`embeds.${fkField}`, () => embedded.EntityInfo.Name, async () => {
+                    // Apply embed fields, passing ownerRecord = entity for @owner: resolution
+                    if (embedRecord.fields && typeof embedRecord.fields === 'object') {
+                        for (const [fName, fVal] of Object.entries(embedRecord.fields)) {
+                            await this.setCompositionField(embedded, fName, fVal, entityDir, batchContext, resolutionCollector, recordProvider, entity, callbacks);
+                        }
                     }
-                }
-                // Recursively apply nested composition on the embedded record
-                await this.applyCompositionAxes(embeddedEntity, embedRecord, embeddedEntity.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                    // Recursively apply nested composition on the embedded record
+                    await this.applyCompositionAxes(embedded, embedRecord, embedded.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                });
             }
         }
         // 2. Collections: owner.Lines.Create() (or load existing by PK into the collection), recurse apply on child entity
@@ -2478,50 +2822,48 @@ export class PushService {
                 const matchedItemSet = new Set();
                 const colConfig = entityConfig?.collections?.[colName];
                 const mode = colConfig?.mode ?? 'upsert';
-                for (const itemData of colItems) {
+                for (const [itemIndex, itemData] of colItems.entries()) {
                     if (!itemData || typeof itemData !== 'object')
                         continue;
-                    let targetChild = null;
-                    // Match by primaryKey if available
-                    if (itemData.primaryKey && Object.keys(itemData.primaryKey).length > 0 && loadedItems.length > 0) {
-                        targetChild = loadedItems.find((child) => {
-                            for (const [pkK, pkV] of Object.entries(itemData.primaryKey)) {
-                                if (String(child.Get(pkK)) !== String(pkV)) {
-                                    return false;
-                                }
+                    await this.withCompositionPath(`collections.${colName}[${itemIndex}]`, () => col.RelatedEntityName, async () => {
+                        let targetChild = null;
+                        // The child's key with `@env:` values resolved; env-sourced key fields are kept for the SQL log
+                        const childKeyCollector = { notes: resolutionCollector.notes, fieldPrefix: 'primaryKey', EnvFields: new Map() };
+                        const childKey = itemData.primaryKey
+                            ? await this.resolveEnvKeyValues(itemData.primaryKey, entityDir, childKeyCollector)
+                            : undefined;
+                        // Match by primaryKey if available
+                        if (childKey && Object.keys(childKey).length > 0 && loadedItems.length > 0) {
+                            targetChild = loadedItems.find((child) => this.childHasKey(child, childKey)) ?? null;
+                        }
+                        if (itemData.deleteRecord?.delete === true) {
+                            // Explicit delete in both modes (§8.1(a) rider 1)
+                            if (targetChild) {
+                                this.queueEnvSqlChildKey(resolutionCollector, targetChild, childKeyCollector);
+                                col.Remove(targetChild);
+                                matchedItemSet.add(targetChild);
                             }
-                            return true;
-                        }) ?? null;
-                    }
-                    if (itemData.deleteRecord?.delete === true) {
-                        // Explicit delete in both modes (§8.1(a) rider 1)
-                        if (targetChild) {
-                            col.Remove(targetChild);
-                            matchedItemSet.add(targetChild);
+                            return;
                         }
-                        continue;
-                    }
-                    if (!targetChild) {
-                        targetChild = await col.Create();
-                        if (itemData.primaryKey) {
-                            for (const [pkK, pkV] of Object.entries(itemData.primaryKey)) {
+                        if (!targetChild) {
+                            targetChild = await col.Create();
+                            for (const [pkK, pkV] of Object.entries(childKey ?? {})) {
                                 targetChild.Set(pkK, pkV);
                             }
                         }
-                    }
-                    else {
-                        matchedItemSet.add(targetChild);
-                    }
-                    // Apply fields with ownerRecord = entity for @owner:Field resolution
-                    if (itemData.fields && typeof itemData.fields === 'object') {
-                        for (const [fName, fVal] of Object.entries(itemData.fields)) {
-                            const processedValue = await this.syncEngine.processFieldValue(fVal, entityDir, null, null, 0, batchContext, resolutionCollector, fName, recordProvider, entity // ownerRecord
-                            );
-                            targetChild.Set(fName, processedValue);
+                        else {
+                            matchedItemSet.add(targetChild);
                         }
-                    }
-                    // Recursively apply nested composition on the collection item
-                    await this.applyCompositionAxes(targetChild, itemData, targetChild.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                        this.queueEnvSqlChildKey(resolutionCollector, targetChild, childKeyCollector);
+                        // Apply fields with ownerRecord = entity for @owner:Field resolution
+                        if (itemData.fields && typeof itemData.fields === 'object') {
+                            for (const [fName, fVal] of Object.entries(itemData.fields)) {
+                                await this.setCompositionField(targetChild, fName, fVal, entityDir, batchContext, resolutionCollector, recordProvider, entity, callbacks);
+                            }
+                        }
+                        // Recursively apply nested composition on the collection item
+                        await this.applyCompositionAxes(targetChild, itemData, targetChild.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                    });
                 }
                 // Handle authoritative mode deletions (§8.1(a))
                 if (mode === 'authoritative' && loadedItems.length > 0) {
@@ -2570,13 +2912,13 @@ export class PushService {
                     throw new Error(`EnsureISAChild returned null for extension on '${entityName}'${subName ? ` (${subName})` : ''}. ` +
                         `Ensure that an EntitySubtypeResolver or Entity.SubtypeSelector is configured, or specify 'entity' in extension.`);
                 }
-                for (const [fName, fVal] of Object.entries(extObj.fields)) {
-                    const processedValue = await this.syncEngine.processFieldValue(fVal, entityDir, null, null, 0, batchContext, resolutionCollector, fName, recordProvider, entity // ownerRecord
-                    );
-                    child.Set(fName, processedValue);
-                }
-                // Recursively apply nested composition on the child extension
-                await this.applyCompositionAxes(child, extObj, child.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                await this.withCompositionPath('extension', () => child.EntityInfo.Name, async () => {
+                    for (const [fName, fVal] of Object.entries(extObj.fields)) {
+                        await this.setCompositionField(child, fName, fVal, entityDir, batchContext, resolutionCollector, recordProvider, entity, callbacks, true);
+                    }
+                    // Recursively apply nested composition on the child extension
+                    await this.applyCompositionAxes(child, extObj, child.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                });
             }
             else {
                 // Map form: { [SubtypeName]: { fields: { ... } } }
@@ -2589,16 +2931,16 @@ export class PushService {
                     if (!child) {
                         throw new Error(`EnsureISAChild returned null for extension subtype '${subKey}' on '${entityName}'.`);
                     }
-                    const subRecord = subVal;
-                    if (subRecord.fields && typeof subRecord.fields === 'object') {
-                        for (const [fName, fVal] of Object.entries(subRecord.fields)) {
-                            const processedValue = await this.syncEngine.processFieldValue(fVal, entityDir, null, null, 0, batchContext, resolutionCollector, fName, recordProvider, entity // ownerRecord
-                            );
-                            child.Set(fName, processedValue);
+                    await this.withCompositionPath(`extension.${subKey}`, () => child.EntityInfo.Name, async () => {
+                        const subRecord = subVal;
+                        if (subRecord.fields && typeof subRecord.fields === 'object') {
+                            for (const [fName, fVal] of Object.entries(subRecord.fields)) {
+                                await this.setCompositionField(child, fName, fVal, entityDir, batchContext, resolutionCollector, recordProvider, entity, callbacks, true);
+                            }
                         }
-                    }
-                    // Recursively apply nested composition on the child extension
-                    await this.applyCompositionAxes(child, subRecord, child.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                        // Recursively apply nested composition on the child extension
+                        await this.applyCompositionAxes(child, subRecord, child.EntityInfo.Name, entityDir, batchContext, resolutionCollector, options, callbacks, entityConfig, recordProvider, depth + 1);
+                    });
                 }
             }
         }