@memberjunction/core 6.2.0-edge.4 → 6.2.0-edge.6

package/dist/generic/databaseProviderBase.js

@@ -1,4 +1,5 @@
 import { ProviderBase } from "./providerBase.js";
+import { UserInfo } from "./securityInfo.js";
 import { EntityFieldTSType, EntityPermissionType } from "./entityInfo.js";
 import { BaseEntityResult, SerializeCloneChangeContext } from "./baseEntity.js";
 import { EntitySaveOptions, EntityDeleteOptions } from "./interfaces.js";
@@ -7,7 +8,8 @@ import { TransactionItem } from "./transactionGroup.js";
 import { CompositeKey } from "./compositeKey.js";
 import { LogError } from "./logging.js";
 import { LocalCacheManager } from "./localCacheManager.js";
-import { EscapeSQLString, FormatBinaryChangeValue, SQLExpressionValidator, StripSQLStringLiterals, uuidv4 } from "@memberjunction/global";
+import { AssertClientInputWritableForUser } from "./entityValueFieldSecurity.js";
+import { FormatBinaryChangeValue, ScanSQLForScreening, SQLExpressionValidator, uuidv4 } from "@memberjunction/global";
 import { GetDialect } from "@memberjunction/sql-dialect";
 // Re-export PlatformSQL types from their canonical location for backward compatibility
 export { IsPlatformSQL } from "./platformSQL.js";
@@ -710,6 +712,110 @@ export class DatabaseProviderBase extends ProviderBase {
     /**************************************************************************/
     // START ---- Record Dependencies
     /**************************************************************************/
+    /**
+     * High-volume system audit/telemetry entities excluded when `options.IncludeSystemEntities === false`.
+     */
+    static { this._systemDependencyEntityNames = new Set([
+        'mj: record changes',
+        'mj: audit logs',
+        'mj: user record logs',
+        'mj: user recordaccess',
+        'mj: entity record documents',
+    ]); }
+    /**
+     * Normalizes `(contextUserOrOptions, options)` so callers can invoke `GetRecordDependencies`
+     * with either `(entityName, key, options)` or `(entityName, key, contextUser, options)`.
+     */
+    resolveRecordDependencyArgs(contextUserOrOptions, options) {
+        if (options !== undefined) {
+            return { contextUser: contextUserOrOptions, options };
+        }
+        if (!contextUserOrOptions) {
+            return { contextUser: undefined, options: undefined };
+        }
+        if (contextUserOrOptions instanceof UserInfo ||
+            'UserRoles' in contextUserOrOptions ||
+            'Email' in contextUserOrOptions) {
+            return { contextUser: contextUserOrOptions, options: undefined };
+        }
+        return { contextUser: undefined, options: contextUserOrOptions };
+    }
+    /**
+     * Determines whether `relatedEntity` should be included in a record dependency scan
+     * based on `options` (`IncludeSystemEntities`, `EntityNames`) and `contextUser` read permissions.
+     */
+    shouldIncludeDependencyEntity(relatedEntity, options, contextUser) {
+        if (!this.shouldIncludeDependencyEntityName(relatedEntity.Name, options)) {
+            return false;
+        }
+        if (contextUser && typeof relatedEntity.CheckPermissions === 'function') {
+            if (!relatedEntity.CheckPermissions(contextUser, EntityPermissionType.Read, false)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Checks name-based dependency filters (`IncludeSystemEntities` and `EntityNames`)
+     * when only the entity name is available.
+     */
+    shouldIncludeDependencyEntityName(relatedEntityName, options) {
+        const normalizedName = relatedEntityName.trim().toLowerCase();
+        if (options?.IncludeSystemEntities === false) {
+            if (DatabaseProviderBase._systemDependencyEntityNames.has(normalizedName)) {
+                return false;
+            }
+        }
+        if (options?.EntityNames && options.EntityNames.length > 0) {
+            const allowed = options.EntityNames.some(n => n.trim().toLowerCase() === normalizedName);
+            if (!allowed) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Resolves an `EntityDependency[] | string` argument to an `EntityDependency[]` array.
+     * Supports passing either a pre-resolved `EntityDependency[]` list or an `entityName` string.
+     */
+    resolveEntityDependencies(entityDependenciesOrName) {
+        if (Array.isArray(entityDependenciesOrName)) {
+            return entityDependenciesOrName;
+        }
+        const eName = entityDependenciesOrName.trim().toLowerCase();
+        const result = [];
+        for (const re of this.Entities) {
+            const relatedFields = re.Fields.filter(f => f.RelatedEntity?.trim().toLowerCase() === eName && (!f.EntityIDFieldName || f.EntityIDFieldName.trim().length === 0));
+            for (const f of relatedFields) {
+                result.push({
+                    EntityName: entityDependenciesOrName,
+                    RelatedEntityName: re.Name,
+                    FieldName: f.Name,
+                });
+            }
+        }
+        return result;
+    }
+    /**
+     * Selects `relatedEntity.BaseTable` when `BaseTable` is defined, the entity is not virtual,
+     * and all referenced columns (`PrimaryKeys` plus `referencedFieldNames`) are physical (`!f.IsVirtual`).
+     * Falls back to `relatedEntity.BaseView` when any referenced column is virtual or `BaseTable` is absent.
+     */
+    resolveDependencySourceObject(relatedEntity, referencedFieldNames) {
+        if (!relatedEntity.BaseTable || relatedEntity.BaseTable.trim().length === 0 || relatedEntity.VirtualEntity) {
+            return relatedEntity.BaseView;
+        }
+        if (relatedEntity.PrimaryKeys?.some(pk => pk.IsVirtual === true)) {
+            return relatedEntity.BaseView;
+        }
+        for (const fieldName of referencedFieldNames) {
+            const field = relatedEntity.Fields?.find(f => f.Name?.trim().toLowerCase() === fieldName?.trim().toLowerCase());
+            if (field?.IsVirtual === true) {
+                return relatedEntity.BaseView;
+            }
+        }
+        return relatedEntity.BaseTable;
+    }
     /**
      * Returns a list of record-level dependencies — records in other entities linked to
      * the specified entity/record via foreign keys (hard links) or EntityIDFieldName
@@ -717,10 +823,12 @@ export class DatabaseProviderBase extends ProviderBase {
      *
      * @param entityName The entity name to check
      * @param compositeKey The primary key(s) of the record
-     * @param contextUser Optional context user
+     * @param contextUser Optional context user (or options when called with 3 args)
+     * @param options Optional filtering options (`LinkType`, `IncludeSystemEntities`, `EntityNames`)
      */
-    async GetRecordDependencies(entityName, compositeKey, contextUser) {
+    async GetRecordDependencies(entityName, compositeKey, contextUser, options) {
         try {
+            const { contextUser: resolvedUser, options: resolvedOptions } = this.resolveRecordDependencyArgs(contextUser, options);
             const recordDependencies = [];
             // Validate up front. Reporting "no dependencies" for an entity we cannot resolve would be
             // indistinguishable, to a caller about to delete a record, from "this record is safe to
@@ -728,19 +836,40 @@ export class DatabaseProviderBase extends ProviderBase {
             if (!this.EntityByName(entityName)) {
                 throw new Error(`Entity ${entityName} not found in metadata`);
             }
-            const entityDependencies = await this.GetEntityDependencies(entityName);
-            // Deliberately NO early return when there are no hard (foreign key) dependents. An entity
-            // whose only dependents are polymorphic EntityID/RecordID links is precisely the case the
-            // soft-link query exists to serve, and returning here skipped building it entirely.
-            const hardSQL = entityDependencies.length > 0 ? this.BuildHardLinkDependencySQL(entityDependencies, compositeKey) : '';
-            const softSQL = this.BuildSoftLinkDependencySQL(entityName, compositeKey);
+            const linkType = resolvedOptions?.LinkType ?? 'both';
+            let hardSQL = '';
+            if (linkType === 'both' || linkType === 'hard') {
+                const rawEntityDependencies = await this.GetEntityDependencies(entityName);
+                const entityDependencies = rawEntityDependencies.filter((dep) => {
+                    const relEntity = this.EntityByName(dep.RelatedEntityName);
+                    if (relEntity) {
+                        return this.shouldIncludeDependencyEntity(relEntity, resolvedOptions, resolvedUser);
+                    }
+                    return this.shouldIncludeDependencyEntityName(dep.RelatedEntityName, resolvedOptions);
+                });
+                // Deliberately NO early return when there are no hard (foreign key) dependents. An entity
+                // whose only dependents are polymorphic EntityID/RecordID links is precisely the case the
+                // soft-link query exists to serve, and returning here skipped building it entirely.
+                hardSQL = entityDependencies.length > 0
+                    ? this.BuildHardLinkDependencySQL(entityDependencies, compositeKey, resolvedOptions, resolvedUser)
+                    : '';
+            }
+            const softSQL = linkType === 'both' || linkType === 'soft'
+                ? this.BuildSoftLinkDependencySQL(entityName, compositeKey, resolvedOptions, resolvedUser)
+                : '';
             const sSQL = [hardSQL, softSQL].filter(s => s.length > 0).join(' UNION ALL ');
             if (!sSQL)
                 return recordDependencies;
-            const result = await this.ExecuteSQL(sSQL, undefined, undefined, contextUser);
+            const result = await this.ExecuteSQL(sSQL, undefined, undefined, resolvedUser);
             if (!result || result.length === 0)
                 return recordDependencies;
-            return this.parseRecordDependencyResults(result);
+            return this.parseRecordDependencyResults(result).filter((dep) => {
+                const relEntity = this.EntityByName(dep.RelatedEntityName);
+                if (relEntity) {
+                    return this.shouldIncludeDependencyEntity(relEntity, resolvedOptions, resolvedUser);
+                }
+                return this.shouldIncludeDependencyEntityName(dep.RelatedEntityName, resolvedOptions);
+            });
         }
         catch (e) {
             LogError(e);
@@ -787,10 +916,16 @@ export class DatabaseProviderBase extends ProviderBase {
                 pkeys[parts[0]] = parts.slice(1).join(CompositeKey.DefaultValueDelimiter);
             });
             depCompositeKey.LoadFromEntityInfoAndRecord(relatedEntityInfo, pkeys);
+            const fieldName = r.FieldName;
+            const fieldInfo = relatedEntityInfo.Fields?.find((f) => f.Name?.trim().toLowerCase() === fieldName?.trim().toLowerCase());
+            const fieldDisplayName = fieldInfo
+                ? (fieldInfo.DisplayNameOrName || fieldInfo.DisplayName || fieldInfo.Name)
+                : undefined;
             recordDependencies.push({
                 EntityName: r.EntityName,
                 RelatedEntityName: r.RelatedEntityName,
-                FieldName: r.FieldName,
+                FieldName: fieldName,
+                FieldDisplayName: fieldDisplayName,
                 PrimaryKey: depCompositeKey,
                 // Both dialects emit this literal on every row of the union: 1/true for the polymorphic
                 // branch, 0/false for the foreign key branch.
@@ -933,8 +1068,23 @@ export class DatabaseProviderBase extends ProviderBase {
         // Always offer the record-change-free form to the SQL logger, not only when the
         // entity tracks record changes: it is also the replay-safe form of a create for
         // migration recordings (MemberJunction/MJ#4503). Execution still uses fullSQL.
-        if (sqlDetails.simpleSQL) {
-            opts.simpleSQLFallback = sqlDetails.simpleSQL;
+        if ('simpleSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'simpleSQLFallback', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.simpleSQL || undefined;
+                },
+            });
+        }
+        if ('literalSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'literalSQL', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.literalSQL || undefined;
+                },
+            });
         }
         return opts;
     }
@@ -946,8 +1096,23 @@ export class DatabaseProviderBase extends ProviderBase {
             isMutation: true,
             description: `Delete ${entity.EntityInfo.Name}`,
         };
-        if (entity.EntityInfo.TrackRecordChanges && sqlDetails.simpleSQL) {
-            opts.simpleSQLFallback = sqlDetails.simpleSQL;
+        if (entity.EntityInfo.TrackRecordChanges && 'simpleSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'simpleSQLFallback', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.simpleSQL || undefined;
+                },
+            });
+        }
+        if ('literalSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'literalSQL', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.literalSQL || undefined;
+                },
+            });
         }
         return opts;
     }
@@ -976,26 +1141,27 @@ export class DatabaseProviderBase extends ProviderBase {
     /**************************************************************************/
     /**
      * Validates a user-provided SQL clause (WHERE, ORDER BY, etc.) to prevent SQL injection.
-     * Checks for forbidden keywords (INSERT, UPDATE, DELETE, EXEC, DROP, UNION, etc.)
-     * and dangerous patterns (comments, semicolons, xp_ prefix).
-     * String literals are stripped before validation to avoid false positives.
+     * The clause is read with this provider's SQL dialect. It is rejected when it contains a
+     * comment, text that cannot be read the way the database reads it, a forbidden keyword
+     * (INSERT, UPDATE, DELETE, EXEC, DROP, UNION, etc.), a semicolon, or an `xp_` name.
+     * String literals and quoted identifiers are not checked for keywords.
      *
      * @param clause The SQL clause to validate
      * @returns true if the clause is safe, false if it contains forbidden patterns
      */
     ValidateUserProvidedSQLClause(clause) {
-        // Remove string literals to avoid false positives.
-        //
-        // 🚨 SECURITY: this uses the SHARED stripper in @memberjunction/global. It must match how
-        // SQL Server / PostgreSQL actually parse literals — see StripSQLStringLiterals for the full
-        // rationale and the backslash-escape bypass it exists to prevent. Do NOT inline a regex here;
-        // an inline copy is exactly how this screen and SQLExpressionValidator drifted apart before.
-        const clauseWithoutStrings = StripSQLStringLiterals(clause);
-        const lowerClause = clauseWithoutStrings.toLowerCase();
+        // 🚨 SECURITY: this uses the SHARED scanner in @memberjunction/global, which reads quotes,
+        // [bracket] and "double-quoted" identifiers, comments, and PostgreSQL E'…' and $$…$$ strings
+        // the way this provider's database does, and refuses text it cannot classify. Do NOT inline
+        // a regex here: this screen and SQLExpressionValidator must read SQL the same way.
+        const scan = ScanSQLForScreening(clause, this.PlatformKey);
+        if (!scan.Classified || scan.CommentCount > 0)
+            return false;
+        const lowerClause = scan.Code.toLowerCase();
         const forbiddenPatterns = [
             /\binsert\b/, /\bupdate\b/, /\bdelete\b/,
             /\bexec\b/, /\bexecute\b/, /\bdrop\b/,
-            /--/, /\/\*/, /\*\//, /\bunion\b/, /\bxp_/, /;/,
+            /--/, /\/\*/, /\*\//, /\bunion\b/, /;/,
             // Time-based blind injection vector — no legitimate filter/order-by clause uses WAITFOR.
             /\bwaitfor\b/,
         ];
@@ -1003,7 +1169,8 @@ export class DatabaseProviderBase extends ProviderBase {
             if (pattern.test(lowerClause))
                 return false;
         }
-        return true;
+        // Extended stored procedures are names, so a quoted name such as [xp_cmdshell] is checked too.
+        return !/\bxp_/.test(scan.CodeWithIdentifiers.toLowerCase());
     }
     /**
      * Checks that the given user has read permissions on the specified entity.
@@ -1052,6 +1219,7 @@ export class DatabaseProviderBase extends ProviderBase {
             const result = validator.validate(agg.expression, {
                 context: 'aggregate',
                 entityFields: fieldNames,
+                Dialect: this.PlatformKey,
             });
             if (!result.valid) {
                 validationErrors.push({
@@ -1138,14 +1306,20 @@ export class DatabaseProviderBase extends ProviderBase {
         }
     }
     /**
-     * Builds the SQL to retrieve the "name" field value for a specific entity record.
-     * Uses QuoteIdentifier/QuoteSchemaAndView for dialect-neutral SQL generation.
+     * Builds the query that reads a record's name fields by its primary key.
+     *
+     * Key columns are written from the entity's metadata and key values are bound as parameters,
+     * so nothing the caller supplies reaches the SQL text. When a user is given, their effective
+     * read row filter (role RLS and API-key row filters) is ANDed in. A caller that gives no user
+     * is trusted server code, as for a single-record Load.
      *
      * @param entityName The entity name
      * @param compositeKey The record's primary key
-     * @returns The SQL query string, or null if the entity has no name field
+     * @param contextUser The acting user, whose read row filter applies
+     * @returns The SQL and its parameter values, or null if the entity has no name field or the
+     * key does not name the entity's primary key fields
      */
-    BuildEntityRecordNameSQL(entityName, compositeKey) {
+    BuildEntityRecordNameSQL(entityName, compositeKey, contextUser) {
         const e = this.EntityByName(entityName);
         if (!e)
             throw new Error('Entity ' + entityName + ' not found');
@@ -1154,30 +1328,44 @@ export class DatabaseProviderBase extends ProviderBase {
             LogError('Entity ' + entityName + ' does not have a NameField, returning null');
             return null;
         }
-        let where = '';
-        for (const pkv of compositeKey.KeyValuePairs) {
-            const pk = e.PrimaryKeys.find((pk) => pk.Name === pkv.FieldName);
-            if (where.length > 0)
-                where += ' AND ';
-            if (pk && pk.NeedsQuotes) {
-                // Key values arrive from remote callers — escape so a quote in the value cannot
-                // break out of the literal (same discipline as CompositeKey.ToWhereClause).
-                where += this.QuoteIdentifier(pkv.FieldName) + "='" + EscapeSQLString(String(pkv.Value)) + "'";
-            }
-            else {
-                // Unquoted (numeric) key column: the value is spliced in bare, so refuse anything
-                // that is not a plain number rather than letting it reach the SQL text.
-                const raw = String(pkv.Value);
-                if (!/^-?\d+(\.\d+)?$/.test(raw)) {
-                    LogError(`BuildEntityRecordNameSQL: non-numeric value provided for numeric key field ${pkv.FieldName} on entity ${entityName}`);
-                    return null;
-                }
-                where += this.QuoteIdentifier(pkv.FieldName) + '=' + raw;
-            }
+        const keyFields = this.RecordNameKeyFieldsOf(e, compositeKey);
+        if (!keyFields) {
+            return null;
+        }
+        const conditions = keyFields.map((f, i) => this.QuoteIdentifier(f.Name) + '=' + this.BuildParameterPlaceholder(i));
+        const rowFilter = contextUser ? e.GetEffectiveRowFilterWhereClause(contextUser, EntityPermissionType.Read, '') : '';
+        if (rowFilter) {
+            conditions.push('(' + rowFilter + ')');
         }
         // SELECT all name fields so InternalGetEntityRecordName can concatenate them
         const selectFields = nameFields.map(f => this.QuoteIdentifier(f.Name)).join(', ');
-        return 'SELECT ' + selectFields + ' FROM ' + this.QuoteSchemaAndView(e.SchemaName, e.BaseView) + ' WHERE ' + where;
+        return {
+            SQL: 'SELECT ' + selectFields + ' FROM ' + this.QuoteSchemaAndView(e.SchemaName, e.BaseView) + ' WHERE ' + conditions.join(' AND '),
+            Parameters: compositeKey.KeyValuePairs.map(kv => kv.Value),
+        };
+    }
+    /**
+     * The primary key field each pair of a record-name key names, in key order. Names match
+     * case-insensitively. Null, after logging, when the key is empty or names a field that is not
+     * one of the entity's primary keys.
+     */
+    RecordNameKeyFieldsOf(entity, compositeKey) {
+        const pairs = compositeKey?.KeyValuePairs ?? [];
+        if (pairs.length === 0) {
+            LogError(`Record name lookup on ${entity.Name}: the key names no fields`);
+            return null;
+        }
+        const keyFields = [];
+        for (const pair of pairs) {
+            const name = String(pair.FieldName ?? '').trim().toLowerCase();
+            const field = entity.PrimaryKeys.find(pk => pk.Name.trim().toLowerCase() === name);
+            if (!field) {
+                LogError(`Record name lookup on ${entity.Name}: ${JSON.stringify(pair.FieldName)} is not a primary key field`);
+                return null;
+            }
+            keyFields.push(field);
+        }
+        return keyFields;
     }
     /**
      * The fields a record's display name is built from, in order: every `IsNameField` field by
@@ -1199,7 +1387,7 @@ export class DatabaseProviderBase extends ProviderBase {
      * because there is nobody to check against.
      */
     CanUserReadRecordName(entity, contextUser) {
-        if (!entity.EnableFieldLevelSecurity) {
+        if (!entity.FieldLevelSecurityApplies) {
             return true;
         }
         if (!contextUser) {
@@ -1210,7 +1398,8 @@ export class DatabaseProviderBase extends ProviderBase {
     }
     /**
      * Retrieves the display name for a single entity record.
-     * Uses BuildEntityRecordNameSQL for dialect-neutral SQL generation.
+     * Uses BuildEntityRecordNameSQL for dialect-neutral SQL generation, which limits the lookup to
+     * rows the acting user's read row filter allows.
      *
      * Answers with an empty string, without querying, when field-level security withholds any of
      * the name fields from the acting user — the same answer as a record that does not exist, so
@@ -1222,9 +1411,9 @@ export class DatabaseProviderBase extends ProviderBase {
             if (entity && !this.CanUserReadRecordName(entity, contextUser)) {
                 return '';
             }
-            const sql = this.BuildEntityRecordNameSQL(entityName, compositeKey);
-            if (sql) {
-                const data = await this.ExecuteSQL(sql, undefined, undefined, contextUser);
+            const query = this.BuildEntityRecordNameSQL(entityName, compositeKey, contextUser);
+            if (query) {
+                const data = await this.ExecuteSQL(query.SQL, query.Parameters, undefined, contextUser);
                 if (data && data.length === 1) {
                     // Concatenate all returned fields with spaces (supports multi-name entities)
                     const values = Object.values(data[0])
@@ -1407,22 +1596,45 @@ export class DatabaseProviderBase extends ProviderBase {
                 }
                 // Step 4: Generate provider-specific SQL
                 const sqlDetails = await this.GenerateSaveSQL(entity, bNewRecord, user, options);
+                const effectiveReturnMode = options?.SaveReturnMode ?? entity.EntityInfo.SaveReturnMode;
+                // Snapshot the entity's in-memory field values at the exact point SQL parameters were bound
+                // (before awaiting ExecuteSQL). If the caller mutates the entity in memory while an async
+                // save is in flight (e.g. BaseEntitySaveQueue pipelining an INSERT + UPDATE), merging
+                // post-await entity.GetAll() would falsely mark those in-flight mutations as clean/saved.
+                const preSaveValues = effectiveReturnMode === 'datestamps_only'
+                    ? entity.GetAll()
+                    : undefined;
+                if (preSaveValues && bNewRecord) {
+                    // On a new record saved with 'datestamps_only', the SP does not query BaseView,
+                    // so virtual non-PK fields (e.g. joined name columns) that were never set stay
+                    // null. Omit them from preSaveValues so finalizeSave marks them NotLoaded=true.
+                    for (const f of entity.EntityInfo.Fields) {
+                        if (f.IsVirtual && !f.IsPrimaryKey && (preSaveValues[f.Name] === null || preSaveValues[f.Name] === undefined)) {
+                            delete preSaveValues[f.Name];
+                        }
+                    }
+                }
                 if (entity.TransactionGroup && !bReplay) {
                     // ---- Transaction Group path ----
                     entity.RaiseReadyForTransaction();
                     suspendRefresh();
                     const extraData = this.GetTransactionExtraData(entity);
-                    if (sqlDetails.simpleSQL) {
-                        extraData.simpleSQLFallback = sqlDetails.simpleSQL;
-                    }
+                    Object.defineProperty(extraData, 'simpleSQLFallback', {
+                        get: () => sqlDetails.simpleSQL,
+                        enumerable: true,
+                        configurable: true,
+                    });
                     extraData.entityName = entity.EntityInfo.Name;
-                    entity.TransactionGroup.AddTransaction(new TransactionItem(entity, entityResult.Type === 'create' ? 'Create' : 'Update', sqlDetails.fullSQL, sqlDetails.parameters ?? null, extraData, (transactionResult, success) => {
+                    entity.TransactionGroup.AddTransaction(new TransactionItem(entity, entityResult.Type === 'create' ? 'Create' : 'Update', sqlDetails.literalSQL ?? sqlDetails.fullSQL, sqlDetails.literalSQL ? null : (sqlDetails.parameters ?? null), extraData, (transactionResult, success) => {
                         resumeRefresh();
                         entityResult.EndedAt = new Date();
                         if (success && transactionResult) {
                             this.OnAfterSaveExecute(entity, user, options, saveContext);
                             entityResult.Success = true;
-                            entityResult.NewValues = this.MapTransactionResultToNewValues(transactionResult);
+                            const mergedResult = preSaveValues
+                                ? { ...preSaveValues, ...transactionResult }
+                                : transactionResult;
+                            entityResult.NewValues = this.MapTransactionResultToNewValues(mergedResult);
                         }
                         else {
                             entityResult.Success = false;
@@ -1448,20 +1660,36 @@ export class DatabaseProviderBase extends ProviderBase {
                     if (result && result.length > 0) {
                         this.OnAfterSaveExecute(entity, user, options, saveContext);
                         entityResult.Success = true;
+                        // In 'datestamps_only' mode, the SP/query returned only the PK and
+                        // __mj_CreatedAt/__mj_UpdatedAt columns. Merge those returned values
+                        // onto the pre-save snapshot so caller-set fields remain intact without
+                        // falsely marking concurrent in-flight mutations as clean.
+                        let mergedRow;
+                        if (preSaveValues) {
+                            mergedRow = { ...preSaveValues, ...result[0] };
+                            Object.defineProperty(mergedRow, '__mj_preMerged', {
+                                value: true,
+                                enumerable: false,
+                                configurable: true,
+                            });
+                        }
+                        else {
+                            mergedRow = result[0];
+                        }
                         // OnSaveCompleted may return patch fields (e.g., updated lat/lng
                         // from geocoding) that need to be merged onto the SP result before
                         // the entity loads the data via finalizeSave().
                         const patches = await this.OnSaveCompleted(entity, sqlDetails, user, options, saveContext);
                         if (patches) {
-                            Object.assign(result[0], patches);
+                            Object.assign(mergedRow, patches);
                         }
-                        return result[0];
+                        return mergedRow;
                     }
                     else {
                         if (bNewRecord)
-                            throw new Error(`SQL Error: Error creating new record, no rows returned from SQL: ${sqlDetails.fullSQL}`);
+                            throw new Error(`SQL Error: Error creating new record, no rows returned from SQL: ${sqlDetails.literalSQL ?? sqlDetails.fullSQL}`);
                         else
-                            throw new Error(`SQL Error: Error updating record, no MATCHING rows found within the database: ${sqlDetails.fullSQL}`);
+                            throw new Error(`SQL Error: Error updating record, no MATCHING rows found within the database: ${sqlDetails.literalSQL ?? sqlDetails.fullSQL}`);
                     }
                 }
             }
@@ -1531,20 +1759,24 @@ export class DatabaseProviderBase extends ProviderBase {
                 // ---- Transaction Group path ----
                 entity.RaiseReadyForTransaction();
                 const extraData = this.GetTransactionExtraData(entity);
-                if (entity.EntityInfo.TrackRecordChanges && sqlDetails.simpleSQL) {
-                    extraData.simpleSQLFallback = sqlDetails.simpleSQL;
+                if (entity.EntityInfo.TrackRecordChanges) {
+                    Object.defineProperty(extraData, 'simpleSQLFallback', {
+                        get: () => sqlDetails.simpleSQL,
+                        enumerable: true,
+                        configurable: true,
+                    });
                 }
                 extraData.entityName = entity.EntityInfo.Name;
-                entity.TransactionGroup.AddTransaction(new TransactionItem(entity, 'Delete', sqlDetails.fullSQL, sqlDetails.parameters ?? null, extraData, (transactionResult, success) => {
+                entity.TransactionGroup.AddTransaction(new TransactionItem(entity, 'Delete', sqlDetails.literalSQL ?? sqlDetails.fullSQL, sqlDetails.literalSQL ? null : (sqlDetails.parameters ?? null), extraData, (transactionResult, success) => {
                     entityResult.EndedAt = new Date();
                     if (success && transactionResult) {
-                        this.OnAfterDeleteExecute(entity, user, options);
-                        for (const key of entity.PrimaryKeys) {
-                            if (key.Value !== transactionResult[key.Name]) {
-                                entityResult.Success = false;
-                                entityResult.Message = 'Transaction failed to commit';
-                            }
+                        // Same key check as the direct path, through the provider's own override
+                        // (PostgreSQL accepts its legacy `_result_id` shape there).
+                        if (!this.ValidateDeleteResult(entity, [transactionResult], entityResult)) {
+                            entityResult.Success = false;
+                            return;
                         }
+                        this.OnAfterDeleteExecute(entity, user, options);
                         entityResult.NewValues = this.MapTransactionResultToNewValues(transactionResult);
                         entityResult.Success = true;
                     }
@@ -1890,6 +2122,11 @@ export class DatabaseProviderBase extends ProviderBase {
         duplicateRun.Set('ProcessingStatus', 'In Progress');
         duplicateRun.Set('ApprovalStatus', 'Pending');
         duplicateRun.Set('SourceListID', listEntity.Get('ID'));
+        // Without a document, the server-side Duplicate Run entity defaults to the entity's
+        // duplicate-detection document.
+        if (params.EntityDocumentID) {
+            duplicateRun.Set('EntityDocumentID', params.EntityDocumentID);
+        }
         const saveResult = await duplicateRun.Save();
         if (!saveResult) {
             throw new Error('Failed to save Duplicate Run Entity');
@@ -1926,6 +2163,13 @@ export class DatabaseProviderBase extends ProviderBase {
                     throw new Error(`Record ${key.ToString()} of ${request.EntityName} has a ${child.ChildEntityName} subtype row; merging records another entity extends is not supported yet`);
             }
         }
+        // The field map is client input applied to the surviving record: a caller with any field
+        // denial may name only fields they can read, and a denied name and an unknown name get the
+        // same refusal. Checked before the merge log and the transaction, so a refusal leaves no trace.
+        if (request.FieldMap?.length) {
+            const values = Object.fromEntries(request.FieldMap.map(f => [f.FieldName, f.Value]));
+            AssertClientInputWritableForUser(e, values, contextUser);
+        }
         const result = {
             Success: false,
             RecordMergeLogID: null,