@memberjunction/metadata-sync 6.2.0-edge.5 → 6.2.0-edge.7

package/dist/services/PushService.js

@@ -27,11 +27,17 @@ import { EnvVariableNotSetError, OptionalEnvMissingError, OptionalEnvSkipMessage
 import { RecordStartLine } from '../lib/json-record-lines.js';
 import { EnvDisplayMask, EnvSourceLabel, ENV_HIDDEN_VALUE, MaskedError } from '../lib/env-display.js';
 import { EnvSqlLog, AddSkippedOptionalEnvField } from '../lib/env-sql-redactor.js';
+import { ComposeStatementTransforms, LookupSqlLog } from '../lib/lookup-sql.js';
+import { ResolveSchemaPlaceholders } from '../lib/schema-placeholders.js';
+import { DatabaseNaturalKeyReader, NaturalKeyConflictMessage, NaturalKeyPreflight } from '../lib/natural-key-preflight.js';
 import { CreatedRecordKeyFields, IsKeyReference, KeyFieldValue } from '../lib/created-record-key.js';
 import { EntityFieldName, RecordFieldNames } from '../lib/entity-field-names.js';
 import { METADATA_KEYWORDS } from '../constants/metadata-keywords.js';
 import { DEFAULT_RECORD_FILE_PATTERN, RECORD_FILE_GLOB_OPTIONS } from '../lib/record-file-glob.js';
+import { FileMayHaveDeletions, RecordsHaveDeletions, TextHasInclude } from '../lib/deletion-prescan.js';
+import { ResolveSqlLogFilter } from '../lib/sql-log-filters.js';
 import { resolveCollectionRelationship } from '@memberjunction/record-graph';
+import { CollectionPromotionResolver } from '../lib/collection-promotion.js';
 function emptyPushTotals() {
     return { created: 0, updated: 0, unchanged: 0, deleted: 0, skipped: 0, deferred: 0, errors: 0 };
 }
@@ -65,6 +71,13 @@ export class PushService {
         this.deferredFileWrites = new Map();
         this.deferredRecords = [];
         this.confirmedCollections = new Set();
+        /** Decides which relatedEntities groups are saved as collection items; created on first use. */
+        this.collectionPromotionResolver = null;
+        /**
+         * Promoted relatedEntities items created by a save, waiting for it to succeed so their new keys
+         * can be written back into the item, the same way a separately saved related record gets its key.
+         */
+        this.pendingPromotedKeys = new Map();
         /** Write mode per entity directory, resolved once at the start of the push. */
         this.directoryModes = new Map();
         /** The directory being processed right now, and how it writes. */
@@ -74,6 +87,10 @@ export class PushService {
         this.runningTotals = emptyPushTotals();
         /** The open SQL log's redactor and lookup-create hook; null when no log is open. */
         this.envSqlLog = null;
+        /** The open SQL log's `@lookup` → subquery rewriter and lookup-create hook; null when no log is open. */
+        this.lookupSqlLog = null;
+        /** Unique keys of each entity's table read for the create pre-flight (#4400), by entity name. */
+        this.uniqueKeyCache = new Map();
         /** Composition child fields waiting for their owner's save, by the owner record's resolution collector. */
         this.pendingEnvSqlChildFields = new WeakMap();
         /** Non-atomic mode: creates and updates that were committed outside the push transaction. */
@@ -370,17 +387,26 @@ export class PushService {
                 // Create the SQL logging session
                 this.sqlLogFilePath = filepath;
                 this.envSqlLog = new EnvSqlLog(this.syncEngine);
+                this.lookupSqlLog = new LookupSqlLog(this.syncEngine);
+                const schemaPlaceholders = ResolveSchemaPlaceholders(this.syncConfig, configManager.GetMJConfig());
+                const logFilter = ResolveSqlLogFilter(this.syncConfig?.sqlLogging);
                 sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
-                    TransformStatement: this.envSqlLog.Redactor.Transform,
+                    // Env values are left out first, then lookup IDs become subqueries; the logger's schema
+                    // placeholder rewrite runs after both, so the subqueries' schema names become placeholders.
+                    TransformStatement: ComposeStatementTransforms(this.envSqlLog.Redactor.Transform, this.lookupSqlLog.Rewriter.Transform),
                     formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
+                    schemaPlaceholders,
+                    // Escaping is tied to migration output, never to whether placeholder rules exist
+                    escapeFlywaySyntax: this.syncConfig?.sqlLogging?.formatAsMigration === true || this.syncConfig?.sqlLogging?.escapeFlywaySyntax === true,
                     description: 'MetadataSync push operation',
                     statementTypes: "mutations",
                     prettyPrint: true,
                     // batchSeparator is intentionally omitted — CreateSqlLogger injects the platform-appropriate
                     // separator automatically (GO for SQL Server, nothing for PostgreSQL) via PlatformBatchSeparator.
-                    filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
-                    filterType: this.syncConfig?.sqlLogging?.filterType,
+                    filterPatterns: logFilter.Patterns,
+                    filterType: logFilter.FilterType,
                     verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
+                    variableBatchThreshold: this.syncConfig?.sqlLogging?.variableBatchThreshold ?? 200,
                 });
                 if (options.verbose) {
                     callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
@@ -456,6 +482,8 @@ export class PushService {
     closeEnvSqlRedactor() {
         this.envSqlLog?.Close();
         this.envSqlLog = null;
+        this.lookupSqlLog?.Close();
+        this.lookupSqlLog = null;
     }
     /**
      * Decide, once and up front, how each entity directory writes: shared by default, isolated where
@@ -829,12 +857,13 @@ export class PushService {
                 // Keep unprocessed data to write back (preserves @file: references)
                 const unprocessedRecords = Array.isArray(rawFileData) ? rawFileData : [rawFileData];
                 const isArray = Array.isArray(rawFileData);
-                // Compute checksum on the RESOLVED content (after @include preprocessing)
-                // so that changes to included files are properly detected.
+                // Compute checksum on the RESOLVED content (after @include preprocessing), with the content
+                // of each @file: target folded in, so a change to an included or referenced file is
+                // detected (#4190). @file: paths resolve against the entity folder, as push resolves them.
                 let cachedChecksum;
                 if (options.incremental && this.stateManager && syncRootDir) {
                     const relativePath = path.relative(syncRootDir, filePath);
-                    cachedChecksum = this.syncEngine.calculateChecksum(fileData);
+                    cachedChecksum = await this.syncEngine.CalculateChecksumWithFileContent(fileData, entityDir);
                     if (!this.stateManager.hasFileChanged(relativePath, cachedChecksum)) {
                         const recordCount = Array.isArray(rawFileData) ? rawFileData.length : 1;
                         if (options.verbose) {
@@ -847,7 +876,9 @@ export class PushService {
                 const records = Array.isArray(fileData) ? fileData : [fileData];
                 // Analyze dependencies and get sorted records
                 const analyzer = new RecordDependencyAnalyzer();
-                const analysisResult = await analyzer.analyzeFileRecords(records, entityConfig.entity);
+                const promotions = this.collectionPromotions();
+                await promotions.Prepare(records, entityConfig.entity);
+                const analysisResult = await analyzer.analyzeFileRecords(records, entityConfig.entity, promotions);
                 for (const flattened of analysisResult.sortedRecords) {
                     flattened.FilePath = filePath;
                 }
@@ -946,7 +977,7 @@ export class PushService {
                 // Uses resolved content (after @include) so included-file changes are tracked.
                 if (this.stateManager && syncRootDir) {
                     const relativePath = path.relative(syncRootDir, filePath);
-                    const checksum = cachedChecksum ?? this.syncEngine.calculateChecksum(fileData);
+                    const checksum = cachedChecksum ?? (await this.syncEngine.CalculateChecksumWithFileContent(fileData, entityDir));
                     this.pendingChecksums.set(relativePath, checksum);
                 }
             }
@@ -1169,7 +1200,10 @@ export class PushService {
         // silent no-ops when seeding a fresh DB from JSON files that already
         // carry stored checksums from another environment.
         // The deferred retry (allowDefer false) never takes this shortcut: the first pass left fields unsaved
-        if (allowDefer && options.incremental && record.sync?.checksum && record.fields && record.primaryKey && !record.deleteRecord) {
+        // A record whose relatedEntities are saved as collection items never takes the shortcut: its
+        // checksum does not cover those items, so an edit to one would be skipped.
+        const hasPromotedGroups = this.collectionPromotions().PromotedGroups(entityName, record).length > 0;
+        if (allowDefer && options.incremental && record.sync?.checksum && record.fields && record.primaryKey && !record.deleteRecord && !hasPromotedGroups) {
             // Use calculateChecksumWithFileContent so @file: reference changes are detected.
             // This reads local files (fast) rather than hitting the DB (slow). The stored
             // checksum was also computed with file content and composition payloads, so the comparison is apples-to-apples.
@@ -1204,7 +1238,7 @@ export class PushService {
         // Check if record exists
         // Process primaryKey values through processFieldValue to resolve @lookup, @parent, etc.
         // Create a resolution collector to track @lookup and @parent resolutions
-        const resolutionCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map() };
+        const resolutionCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map(), LookupSources: new Map() };
         // Reads the same EnvFields map, so fields recorded later are masked too
         const envMask = new EnvDisplayMask(resolutionCollector.EnvFields);
         let resolvedPrimaryKey;
@@ -1500,6 +1534,11 @@ export class PushService {
                 status: 'unchanged',
             };
         }
+        // A literal key that missed can still collide on a unique key with a row this database made
+        // under another ID: report that row instead of a constraint violation (#4400).
+        if (isNew && resolvedPrimaryKey && Object.keys(resolvedPrimaryKey).length > 0) {
+            await this.checkNaturalKey(entity, resolvedPrimaryKey, envMask, recordProvider);
+        }
         // A dry run stops here, after the same change detection a real push uses, so an
         // in-sync record reports unchanged above rather than "would update" (#4529).
         if (options.dryRun) {
@@ -1531,6 +1570,8 @@ export class PushService {
         }
         this.registerEnvSqlFields(entity, resolutionCollector.EnvFields, isNew || alwaysPush);
         this.registerPendingEnvSqlChildFields(resolutionCollector);
+        // Key and foreign-key values from @lookup are logged as subqueries on the natural key (#5626)
+        this.lookupSqlLog?.Rewriter.RegisterEntityLookups(entity, resolutionCollector.LookupSources);
         // Save the record with detailed error logging
         const keyBeforeSave = new Map(entity.PrimaryKeys.map((pk) => [pk.Name, entity.Get(pk.Name)]));
         const recordName = entity.Get('Name');
@@ -1697,6 +1738,7 @@ export class PushService {
             // Return 'deferred' status - it's saved but incomplete
             // We don't return early here because we still want to update primaryKey and sync metadata
         }
+        this.writeBackPromotedKeys(record);
         // Update primaryKey for new records
         if (isNew) {
             const entityInfo = this.syncEngine.getEntityInfo(entityName);
@@ -1891,7 +1933,9 @@ export class PushService {
         const keyCollector = { notes: [], fieldPrefix: 'primaryKey', EnvFields: new Map() };
         const resolvedKey = await this.resolveEnvKeyValues(record.primaryKey, entityDir, keyCollector);
         const keyMask = new EnvDisplayMask(keyCollector.EnvFields);
-        const existingEntity = await this.syncEngine.loadEntity(entityName, resolvedKey);
+        // Read through the push transaction's provider, not the preload cache: a row an earlier delete's
+        // cascade removed in this push is still in the cache until commit (#4935).
+        const existingEntity = await this.syncEngine.LoadEntityFromDatabase(entityName, resolvedKey, this.transactionProvider());
         // Check if the deletion has already been processed
         if (record.deleteRecord?.deletedAt) {
             // Verify if record still exists in THIS database
@@ -1909,19 +1953,7 @@ export class PushService {
             // Fall through to deletion logic
         }
         if (!existingEntity) {
-            const pkDisplay = Object.entries(record.primaryKey)
-                .map(([key, value]) => `${key}=${this.fileKeyText(value)}`)
-                .join(', ');
-            const warning = `Record not found for deletion: ${entityName} with primaryKey {${pkDisplay}}`;
-            this.warnings.push(warning);
-            callbacks?.onWarn?.(warning);
-            // Mark as deleted anyway since it doesn't exist
-            if (!record.deleteRecord) {
-                record.deleteRecord = { delete: true };
-            }
-            record.deleteRecord.deletedAt = undefined; // Indicate it was not found
-            record.deleteRecord.notFound = true;
-            return { status: 'skipped', isDuplicate: false };
+            return this.skipMissingDelete(flattenedRecord, 'Record not found for deletion', callbacks);
         }
         // Key values and free error text can hold a value of any variable push resolved
         const errorTextMask = EnvDisplayMask.ForVariables(this.syncEngine.ResolvedEnvVariables);
@@ -1971,6 +2003,10 @@ export class PushService {
         try {
             const deleteResult = await existingEntity.Delete();
             if (!deleteResult) {
+                // A row already gone (removed by an earlier delete's cascade) is the intended end state
+                if (!(await this.rowStillExists(entityName, resolvedKey))) {
+                    return this.skipMissingDelete(flattenedRecord, 'Record was already gone when it was deleted (an earlier delete in this push may have cascaded to it)', callbacks);
+                }
                 // Check the LatestResult for error details
                 const errorMessage = errorTextMask.MaskText(existingEntity.LatestResult?.CompleteMessage || 'Unknown error');
                 const errorDetails = errorTextMask.MaskText(existingEntity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || '');
@@ -2010,6 +2046,9 @@ export class PushService {
             return { status: 'deleted', isDuplicate: false };
         }
         catch (deleteError) {
+            if (!(await this.rowStillExists(entityName, resolvedKey))) {
+                return this.skipMissingDelete(flattenedRecord, 'Record was already gone when it was deleted (an earlier delete in this push may have cascaded to it)', callbacks);
+            }
             console.error(`\n❌ DELETE EXCEPTION for ${entityName}`);
             console.error(`   Primary Key: {${primaryKeyDisplay.join(', ')}}`);
             console.error(`   Error: ${errorTextMask.MaskText(String(deleteError.message || deleteError))}`);
@@ -2024,6 +2063,61 @@ export class PushService {
             throw MaskedError(deleteError, errorTextMask);
         }
     }
+    /**
+     * A delete whose row is not in the database: a warning, the record marked not found, and a skip.
+     * The database is already in the state the delete asks for.
+     */
+    skipMissingDelete(flattenedRecord, reason, callbacks) {
+        const { record, entityName } = flattenedRecord;
+        const pkDisplay = Object.entries(record.primaryKey ?? {})
+            .map(([key, value]) => `${key}=${this.fileKeyText(value)}`)
+            .join(', ');
+        const warning = `${reason}: ${entityName} with primaryKey {${pkDisplay}}`;
+        this.warnings.push(warning);
+        callbacks?.onWarn?.(warning);
+        // Mark as deleted anyway since it doesn't exist
+        if (!record.deleteRecord) {
+            record.deleteRecord = { delete: true };
+        }
+        record.deleteRecord.deletedAt = undefined; // Indicate it was not found
+        record.deleteRecord.notFound = true;
+        return { status: 'skipped', isDuplicate: false };
+    }
+    /**
+     * True when the row is still in the database, read through the push transaction. A failed read
+     * counts as present, so the original delete error is reported.
+     */
+    async rowStillExists(entityName, primaryKey) {
+        try {
+            return (await this.syncEngine.LoadEntityFromDatabase(entityName, primaryKey, this.transactionProvider())) !== null;
+        }
+        catch {
+            return true;
+        }
+    }
+    /**
+     * Throws when creating `entity` with the key the record declares would collide on a unique key
+     * with an existing row, naming that row's key. See natural-key-preflight.ts.
+     */
+    async checkNaturalKey(entity, resolvedPrimaryKey, envMask, recordProvider) {
+        const host = this.hostProvider();
+        if (typeof host?.ExecuteSQL !== 'function') {
+            return; // no SQL provider, so no catalog to read unique keys from
+        }
+        // Rows are found on the record's provider, so the lookup sees what its transaction wrote
+        const reader = new DatabaseNaturalKeyReader(host, this.contextUser, recordProvider ?? this.transactionProvider(), this.uniqueKeyCache);
+        const conflict = await new NaturalKeyPreflight(reader).Check(entity);
+        if (conflict) {
+            const recordKey = Object.entries(resolvedPrimaryKey)
+                .map(([field, value]) => `${field}=${envMask.NewValue(field, String(value))}`)
+                .join(', ');
+            throw new Error(NaturalKeyConflictMessage(entity.EntityInfo.Name, recordKey, conflict));
+        }
+    }
+    /** The provider that owns the push transaction, as the metadata provider a read joins it through. */
+    transactionProvider() {
+        return this.hostProvider();
+    }
     /**
      * Prompt user for confirmation before proceeding with push operation
      * This happens after deletion audit but before any database operations
@@ -2123,20 +2217,14 @@ export class PushService {
                 dot: true,
                 ignore: ['**/node_modules/**', '**/.mj-*.json']
             });
-            // Quick scan for delete directives without full processing
+            // Quick scan for delete markers at any depth. A file that uses @include falls through to
+            // the full audit, which expands includes before looking.
             for (const filePath of files) {
                 try {
                     const content = await fs.readFile(filePath, 'utf-8');
-                    // Fast string check for delete directives
-                    if (content.includes('"delete"') && content.includes('true')) {
-                        // More precise check - parse JSON to confirm
-                        const data = JSON.parse(content);
-                        const records = Array.isArray(data) ? data : [data];
-                        const hasDelete = records.some((r) => r.deleteRecord?.delete === true);
-                        if (hasDelete) {
-                            hasAnyDeletions = true;
-                            break;
-                        }
+                    if (FileMayHaveDeletions(content)) {
+                        hasAnyDeletions = true;
+                        break;
                     }
                 }
                 catch (error) {
@@ -2180,9 +2268,7 @@ export class PushService {
                     const rawFileData = await fs.readJson(filePath);
                     // Handle @include directives if present
                     let fileData = rawFileData;
-                    const jsonString = JSON.stringify(rawFileData);
-                    const hasIncludes = jsonString.includes('"@include"') || jsonString.includes('"@include.');
-                    if (hasIncludes) {
+                    if (TextHasInclude(JSON.stringify(rawFileData))) {
                         const jsonPreprocessor = new JsonPreprocessor();
                         fileData = await jsonPreprocessor.processFile(filePath);
                     }
@@ -2317,23 +2403,7 @@ export class PushService {
      * Recursively check if any records in an array (including nested relatedEntities) have deletions
      */
     hasAnyDeletions(records) {
-        for (const record of records) {
-            // Check this record
-            if (record.deleteRecord?.delete === true) {
-                return true;
-            }
-            // Check nested related entities recursively
-            if (record.relatedEntities) {
-                for (const relatedRecords of Object.values(record.relatedEntities)) {
-                    if (Array.isArray(relatedRecords)) {
-                        if (this.hasAnyDeletions(relatedRecords)) {
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-        return false;
+        return RecordsHaveDeletions(records);
     }
     /**
      * Process deferred records that had lookup failures during initial processing.
@@ -2554,7 +2624,8 @@ export class PushService {
             ? resolutionCollector
             : { notes: resolutionCollector.notes, fieldPrefix: resolutionCollector.fieldPrefix, EnvFields: new Map() };
         try {
-            const value = await this.syncEngine.processFieldValue(rawValue, entityDir, null, null, 0, batchContext, childCollector, fieldName, recordProvider, ownerRecord);
+            // `@parent:` in a composition child refers to its owner, as it does for a related record.
+            const value = await this.syncEngine.processFieldValue(rawValue, entityDir, ownerRecord, null, 0, batchContext, childCollector, fieldName, recordProvider, ownerRecord);
             target.Set(fieldName, value);
             const variables = childCollector.EnvFields?.get(fieldName);
             if (variables) {
@@ -2621,6 +2692,53 @@ export class PushService {
         for (const warning of this.envSqlLog?.Redactor.Warnings(sqlLogPath) ?? []) {
             this.addWarning(warning, callbacks);
         }
+        for (const warning of this.lookupSqlLog?.Rewriter.Warnings() ?? []) {
+            this.addWarning(warning, callbacks);
+        }
+    }
+    /** The resolver deciding which relatedEntities groups are saved as collection items. */
+    collectionPromotions() {
+        if (!this.collectionPromotionResolver) {
+            this.collectionPromotionResolver = new CollectionPromotionResolver(() => this.syncEngine.GetProvider(), this.contextUser);
+        }
+        return this.collectionPromotionResolver;
+    }
+    /**
+     * The collections to apply to a record: its own `collections`, plus, for a flattened record
+     * (depth 0), the `relatedEntities` groups promoted to collection items. The record itself is not
+     * changed, so its file keeps the format the author wrote.
+     *
+     * @returns The collections and the set of promoted items among them, or `null` when there are none.
+     */
+    collectionsToApply(record, entityName, depth) {
+        const collections = record.collections && typeof record.collections === 'object' ? { ...record.collections } : {};
+        const promotedItems = new Set();
+        if (depth === 0) {
+            for (const group of this.collectionPromotions().PromotedGroups(entityName, record)) {
+                const name = group.Promotion.CollectionName;
+                collections[name] = [...(collections[name] ?? []), ...group.Items];
+                group.Items.forEach((item) => promotedItems.add(item));
+            }
+        }
+        return Object.keys(collections).length > 0 ? { Collections: collections, PromotedItems: promotedItems } : null;
+    }
+    /** Remembers a promoted item created without a key, to receive its key once the save succeeds. */
+    queuePromotedKey(record, item, child) {
+        const pending = this.pendingPromotedKeys.get(record) ?? [];
+        pending.push({ Item: item, Child: child });
+        this.pendingPromotedKeys.set(record, pending);
+    }
+    /** Writes the saved keys of a record's new promoted items back into the items. */
+    writeBackPromotedKeys(record) {
+        const pending = this.pendingPromotedKeys.get(record);
+        this.pendingPromotedKeys.delete(record);
+        for (const { Item, Child } of pending ?? []) {
+            const primaryKey = {};
+            for (const pk of Child.EntityInfo.PrimaryKeys) {
+                primaryKey[pk.Name] = Child.Get(pk.Name);
+            }
+            Item.primaryKey = primaryKey;
+        }
     }
     /**
      * Applies first-class composition axes onto the entity before Save (§5):
@@ -2682,8 +2800,9 @@ export class PushService {
             }
         }
         // 2. Collections: owner.Lines.Create() (or load existing by PK into the collection), recurse apply on child entity
-        if (record.collections && typeof record.collections === 'object') {
-            for (const [colName, colItems] of Object.entries(record.collections)) {
+        const collectionsToApply = this.collectionsToApply(record, entityName, depth);
+        if (collectionsToApply) {
+            for (const [colName, colItems] of Object.entries(collectionsToApply.Collections)) {
                 if (!Array.isArray(colItems)) {
                     throw new Error(`Collection "${colName}" in ${entityName} must be an array of records. ` +
                         `Per-record mode wrappers (e.g. {"mode": "authoritative", "items": [...]}) are forbidden; mode is directory-level only.`);
@@ -2770,6 +2889,9 @@ export class PushService {
                             for (const [pkK, pkV] of Object.entries(childKey ?? {})) {
                                 targetChild.Set(pkK, pkV);
                             }
+                            if (collectionsToApply.PromotedItems.has(itemData) && !childKey) {
+                                this.queuePromotedKey(record, itemData, targetChild);
+                            }
                         }
                         else {
                             matchedItemSet.add(targetChild);