@memberjunction/core 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/generic/databaseProviderBase.js

@@ -6,7 +6,8 @@ import { DispatchRemoteOperationInProcess } from "./remoteOperationDispatch.js";
 import { TransactionItem } from "./transactionGroup.js";
 import { CompositeKey } from "./compositeKey.js";
 import { LogError } from "./logging.js";
-import { EscapeSQLString, FormatBinaryChangeValue, SQLExpressionValidator, StripSQLStringLiterals, uuidv4 } from "@memberjunction/global";
+import { LocalCacheManager } from "./localCacheManager.js";
+import { FormatBinaryChangeValue, ScanSQLForScreening, SQLExpressionValidator, uuidv4 } from "@memberjunction/global";
 import { GetDialect } from "@memberjunction/sql-dialect";
 // Re-export PlatformSQL types from their canonical location for backward compatibility
 export { IsPlatformSQL } from "./platformSQL.js";
@@ -82,10 +83,13 @@ export class DatabaseProviderBase extends ProviderBase {
         return 0;
     }
     /**
-     * Public nesting depth. 0 = no ambient TX. Join-TX callers (accounting
-     * CreateJournalEntries) must read this, not `IsInTransaction` (SQL Server
-     * leaves that false). Deprecated camelCase `transactionDepth` alias ships
-     * for one release.
+     * Public nesting depth of the transaction open on this instance; 0 = none. Read this, not
+     * `IsInTransaction` (SQL Server leaves that false), when you need to know whether a transaction
+     * is open at all. The depth is instance-wide: on a shared provider `TransactionDepth > 0` does
+     * NOT mean the caller's statements run in that transaction — only the async scope that began it
+     * is part of it (#4786). Code that needs to be inside a transaction should simply call
+     * {@link BeginTransaction} / {@link BeginEntityTransaction}: joining one the caller is already in
+     * is automatic. Deprecated camelCase `transactionDepth` alias ships for one release.
      */
     get TransactionDepth() {
         return this.CurrentTransactionDepth;
@@ -114,6 +118,11 @@ export class DatabaseProviderBase extends ProviderBase {
                 await this.ResetTransactionState();
             }
         }
+        // Belt and braces: whatever route the rollback took, this instance is going away, so it
+        // must not leave an entity-event batch behind. A batch whose owner is collected is
+        // unreachable while its entities keep counting as pending, which makes every cached read
+        // of them miss for the life of the process.
+        await LocalCacheManager.Instance.AbandonEntityEventBatch(this);
     }
     /** @deprecated Use {@link TransactionDepth}. */
     get transactionDepth() {
@@ -128,15 +137,15 @@ export class DatabaseProviderBase extends ProviderBase {
         /* no-op */
     }
     /**
-     * Run `task` once the ambient transaction on this provider has committed, or now if there is
-     * no ambient transaction.
+     * Run `task` once the caller's transaction on this provider (the one its async scope belongs
+     * to) has committed, or now if the caller is in no transaction on this provider.
      *
      * Use it for side effects that must only happen for work that is actually durable — queueing
      * background jobs, firing deferred entity actions — and must not run *inside* the caller's
      * transaction. Providers that track transactions (see `GenericDatabaseProvider`) queue the task
-     * while a transaction is open, run queued tasks in registration order after the **outermost**
-     * commit succeeds (once the transaction lock is released, so a task may open its own
-     * transaction), and discard them — without running them — when the transaction rolls back,
+     * while the caller's transaction is open, run queued tasks in registration order after the
+     * **outermost** commit succeeds (once the transaction lock is released, so a task may open its
+     * own transaction), and discard them — without running them — when the transaction rolls back,
      * fails to commit, or is abandoned.
      *
      * This default is for providers that do not track transactions: the task starts immediately.
@@ -163,8 +172,9 @@ export class DatabaseProviderBase extends ProviderBase {
         void this.RunPostCommitTaskSafely(task, description);
     }
     /**
-     * Snapshot of the transaction frames open right now, for a later {@link RunAfterCommit}.
-     * Synchronous by contract — call it before the first `await` of the code that causes the work.
+     * Snapshot of the caller's transaction frames on this provider (the transaction its async scope
+     * belongs to), for a later {@link RunAfterCommit}. Synchronous by contract — call it before the
+     * first `await` of the code that causes the work.
      *
      * @returns `undefined` when no transaction is open. This default never tracks transactions, so
      *          it always returns `undefined`.
@@ -207,6 +217,11 @@ export class DatabaseProviderBase extends ProviderBase {
      * physical transaction on the same pool, blind to any transaction the caller had already
      * started.
      *
+     * The transaction belongs to the async scope that calls this: that code and everything it starts
+     * run on the transaction; unrelated callers on the same provider instance run on the pool
+     * (GenericDatabaseProvider, #4786). Call it from the function that performs the writes — a begin
+     * reached only after an `await` inside a helper does not include the helper's caller.
+     *
      * The returned scope is **settle-once**: the first `Commit()` or `Rollback()` wins and later
      * calls are no-ops, so `try { ...; Commit() } catch { Rollback() }` is safe even when the work
      * already unwound its own scope.
@@ -222,6 +237,10 @@ export class DatabaseProviderBase extends ProviderBase {
         const isNested = this.IsInTransaction || this.CurrentTransactionDepth > 0;
         await this.BeginTransaction();
         const depthAtBegin = this.CurrentTransactionDepth;
+        // Cache maintenance for the saves in this scope waits for the outermost settle: one rewrite
+        // per cached slot on commit instead of one per save, and nothing written for work that is
+        // rolled back. Nested scopes join the same batch.
+        LocalCacheManager.Instance.BeginEntityEventBatch(this);
         let settled = false;
         const settle = async (commit) => {
             if (settled) {
@@ -244,11 +263,18 @@ export class DatabaseProviderBase extends ProviderBase {
                     `work. Do not run concurrent transactional saves on a shared provider instance — use ` +
                     `per-request providers (as MJServer does) or serialize the units of work.`);
             }
-            if (commit) {
-                await this.CommitTransaction();
+            let settledAsCommit = false;
+            try {
+                if (commit) {
+                    await this.CommitTransaction();
+                    settledAsCommit = true;
+                }
+                else {
+                    await this.RollbackTransaction();
+                }
             }
-            else {
-                await this.RollbackTransaction();
+            finally {
+                await LocalCacheManager.Instance.EndEntityEventBatch(this, settledAsCommit);
             }
         };
         return {
@@ -907,8 +933,23 @@ export class DatabaseProviderBase extends ProviderBase {
         // Always offer the record-change-free form to the SQL logger, not only when the
         // entity tracks record changes: it is also the replay-safe form of a create for
         // migration recordings (MemberJunction/MJ#4503). Execution still uses fullSQL.
-        if (sqlDetails.simpleSQL) {
-            opts.simpleSQLFallback = sqlDetails.simpleSQL;
+        if ('simpleSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'simpleSQLFallback', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.simpleSQL || undefined;
+                },
+            });
+        }
+        if ('literalSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'literalSQL', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.literalSQL || undefined;
+                },
+            });
         }
         return opts;
     }
@@ -920,8 +961,23 @@ export class DatabaseProviderBase extends ProviderBase {
             isMutation: true,
             description: `Delete ${entity.EntityInfo.Name}`,
         };
-        if (entity.EntityInfo.TrackRecordChanges && sqlDetails.simpleSQL) {
-            opts.simpleSQLFallback = sqlDetails.simpleSQL;
+        if (entity.EntityInfo.TrackRecordChanges && 'simpleSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'simpleSQLFallback', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.simpleSQL || undefined;
+                },
+            });
+        }
+        if ('literalSQL' in sqlDetails) {
+            Object.defineProperty(opts, 'literalSQL', {
+                enumerable: true,
+                configurable: true,
+                get() {
+                    return sqlDetails.literalSQL || undefined;
+                },
+            });
         }
         return opts;
     }
@@ -950,26 +1006,27 @@ export class DatabaseProviderBase extends ProviderBase {
     /**************************************************************************/
     /**
      * Validates a user-provided SQL clause (WHERE, ORDER BY, etc.) to prevent SQL injection.
-     * Checks for forbidden keywords (INSERT, UPDATE, DELETE, EXEC, DROP, UNION, etc.)
-     * and dangerous patterns (comments, semicolons, xp_ prefix).
-     * String literals are stripped before validation to avoid false positives.
+     * The clause is read with this provider's SQL dialect. It is rejected when it contains a
+     * comment, text that cannot be read the way the database reads it, a forbidden keyword
+     * (INSERT, UPDATE, DELETE, EXEC, DROP, UNION, etc.), a semicolon, or an `xp_` name.
+     * String literals and quoted identifiers are not checked for keywords.
      *
      * @param clause The SQL clause to validate
      * @returns true if the clause is safe, false if it contains forbidden patterns
      */
     ValidateUserProvidedSQLClause(clause) {
-        // Remove string literals to avoid false positives.
-        //
-        // 🚨 SECURITY: this uses the SHARED stripper in @memberjunction/global. It must match how
-        // SQL Server / PostgreSQL actually parse literals — see StripSQLStringLiterals for the full
-        // rationale and the backslash-escape bypass it exists to prevent. Do NOT inline a regex here;
-        // an inline copy is exactly how this screen and SQLExpressionValidator drifted apart before.
-        const clauseWithoutStrings = StripSQLStringLiterals(clause);
-        const lowerClause = clauseWithoutStrings.toLowerCase();
+        // 🚨 SECURITY: this uses the SHARED scanner in @memberjunction/global, which reads quotes,
+        // [bracket] and "double-quoted" identifiers, comments, and PostgreSQL E'…' and $$…$$ strings
+        // the way this provider's database does, and refuses text it cannot classify. Do NOT inline
+        // a regex here: this screen and SQLExpressionValidator must read SQL the same way.
+        const scan = ScanSQLForScreening(clause, this.PlatformKey);
+        if (!scan.Classified || scan.CommentCount > 0)
+            return false;
+        const lowerClause = scan.Code.toLowerCase();
         const forbiddenPatterns = [
             /\binsert\b/, /\bupdate\b/, /\bdelete\b/,
             /\bexec\b/, /\bexecute\b/, /\bdrop\b/,
-            /--/, /\/\*/, /\*\//, /\bunion\b/, /\bxp_/, /;/,
+            /--/, /\/\*/, /\*\//, /\bunion\b/, /;/,
             // Time-based blind injection vector — no legitimate filter/order-by clause uses WAITFOR.
             /\bwaitfor\b/,
         ];
@@ -977,7 +1034,8 @@ export class DatabaseProviderBase extends ProviderBase {
             if (pattern.test(lowerClause))
                 return false;
         }
-        return true;
+        // Extended stored procedures are names, so a quoted name such as [xp_cmdshell] is checked too.
+        return !/\bxp_/.test(scan.CodeWithIdentifiers.toLowerCase());
     }
     /**
      * Checks that the given user has read permissions on the specified entity.
@@ -1026,6 +1084,7 @@ export class DatabaseProviderBase extends ProviderBase {
             const result = validator.validate(agg.expression, {
                 context: 'aggregate',
                 entityFields: fieldNames,
+                Dialect: this.PlatformKey,
             });
             if (!result.valid) {
                 validationErrors.push({
@@ -1112,14 +1171,20 @@ export class DatabaseProviderBase extends ProviderBase {
         }
     }
     /**
-     * Builds the SQL to retrieve the "name" field value for a specific entity record.
-     * Uses QuoteIdentifier/QuoteSchemaAndView for dialect-neutral SQL generation.
+     * Builds the query that reads a record's name fields by its primary key.
+     *
+     * Key columns are written from the entity's metadata and key values are bound as parameters,
+     * so nothing the caller supplies reaches the SQL text. When a user is given, their effective
+     * read row filter (role RLS and API-key row filters) is ANDed in. A caller that gives no user
+     * is trusted server code, as for a single-record Load.
      *
      * @param entityName The entity name
      * @param compositeKey The record's primary key
-     * @returns The SQL query string, or null if the entity has no name field
+     * @param contextUser The acting user, whose read row filter applies
+     * @returns The SQL and its parameter values, or null if the entity has no name field or the
+     * key does not name the entity's primary key fields
      */
-    BuildEntityRecordNameSQL(entityName, compositeKey) {
+    BuildEntityRecordNameSQL(entityName, compositeKey, contextUser) {
         const e = this.EntityByName(entityName);
         if (!e)
             throw new Error('Entity ' + entityName + ' not found');
@@ -1128,30 +1193,44 @@ export class DatabaseProviderBase extends ProviderBase {
             LogError('Entity ' + entityName + ' does not have a NameField, returning null');
             return null;
         }
-        let where = '';
-        for (const pkv of compositeKey.KeyValuePairs) {
-            const pk = e.PrimaryKeys.find((pk) => pk.Name === pkv.FieldName);
-            if (where.length > 0)
-                where += ' AND ';
-            if (pk && pk.NeedsQuotes) {
-                // Key values arrive from remote callers — escape so a quote in the value cannot
-                // break out of the literal (same discipline as CompositeKey.ToWhereClause).
-                where += this.QuoteIdentifier(pkv.FieldName) + "='" + EscapeSQLString(String(pkv.Value)) + "'";
-            }
-            else {
-                // Unquoted (numeric) key column: the value is spliced in bare, so refuse anything
-                // that is not a plain number rather than letting it reach the SQL text.
-                const raw = String(pkv.Value);
-                if (!/^-?\d+(\.\d+)?$/.test(raw)) {
-                    LogError(`BuildEntityRecordNameSQL: non-numeric value provided for numeric key field ${pkv.FieldName} on entity ${entityName}`);
-                    return null;
-                }
-                where += this.QuoteIdentifier(pkv.FieldName) + '=' + raw;
-            }
+        const keyFields = this.RecordNameKeyFieldsOf(e, compositeKey);
+        if (!keyFields) {
+            return null;
+        }
+        const conditions = keyFields.map((f, i) => this.QuoteIdentifier(f.Name) + '=' + this.BuildParameterPlaceholder(i));
+        const rowFilter = contextUser ? e.GetEffectiveRowFilterWhereClause(contextUser, EntityPermissionType.Read, '') : '';
+        if (rowFilter) {
+            conditions.push('(' + rowFilter + ')');
         }
         // SELECT all name fields so InternalGetEntityRecordName can concatenate them
         const selectFields = nameFields.map(f => this.QuoteIdentifier(f.Name)).join(', ');
-        return 'SELECT ' + selectFields + ' FROM ' + this.QuoteSchemaAndView(e.SchemaName, e.BaseView) + ' WHERE ' + where;
+        return {
+            SQL: 'SELECT ' + selectFields + ' FROM ' + this.QuoteSchemaAndView(e.SchemaName, e.BaseView) + ' WHERE ' + conditions.join(' AND '),
+            Parameters: compositeKey.KeyValuePairs.map(kv => kv.Value),
+        };
+    }
+    /**
+     * The primary key field each pair of a record-name key names, in key order. Names match
+     * case-insensitively. Null, after logging, when the key is empty or names a field that is not
+     * one of the entity's primary keys.
+     */
+    RecordNameKeyFieldsOf(entity, compositeKey) {
+        const pairs = compositeKey?.KeyValuePairs ?? [];
+        if (pairs.length === 0) {
+            LogError(`Record name lookup on ${entity.Name}: the key names no fields`);
+            return null;
+        }
+        const keyFields = [];
+        for (const pair of pairs) {
+            const name = String(pair.FieldName ?? '').trim().toLowerCase();
+            const field = entity.PrimaryKeys.find(pk => pk.Name.trim().toLowerCase() === name);
+            if (!field) {
+                LogError(`Record name lookup on ${entity.Name}: ${JSON.stringify(pair.FieldName)} is not a primary key field`);
+                return null;
+            }
+            keyFields.push(field);
+        }
+        return keyFields;
     }
     /**
      * The fields a record's display name is built from, in order: every `IsNameField` field by
@@ -1184,7 +1263,8 @@ export class DatabaseProviderBase extends ProviderBase {
     }
     /**
      * Retrieves the display name for a single entity record.
-     * Uses BuildEntityRecordNameSQL for dialect-neutral SQL generation.
+     * Uses BuildEntityRecordNameSQL for dialect-neutral SQL generation, which limits the lookup to
+     * rows the acting user's read row filter allows.
      *
      * Answers with an empty string, without querying, when field-level security withholds any of
      * the name fields from the acting user — the same answer as a record that does not exist, so
@@ -1196,9 +1276,9 @@ export class DatabaseProviderBase extends ProviderBase {
             if (entity && !this.CanUserReadRecordName(entity, contextUser)) {
                 return '';
             }
-            const sql = this.BuildEntityRecordNameSQL(entityName, compositeKey);
-            if (sql) {
-                const data = await this.ExecuteSQL(sql, undefined, undefined, contextUser);
+            const query = this.BuildEntityRecordNameSQL(entityName, compositeKey, contextUser);
+            if (query) {
+                const data = await this.ExecuteSQL(query.SQL, query.Parameters, undefined, contextUser);
                 if (data && data.length === 1) {
                     // Concatenate all returned fields with spaces (supports multi-name entities)
                     const values = Object.values(data[0])
@@ -1381,22 +1461,45 @@ export class DatabaseProviderBase extends ProviderBase {
                 }
                 // Step 4: Generate provider-specific SQL
                 const sqlDetails = await this.GenerateSaveSQL(entity, bNewRecord, user, options);
+                const effectiveReturnMode = options?.SaveReturnMode ?? entity.EntityInfo.SaveReturnMode;
+                // Snapshot the entity's in-memory field values at the exact point SQL parameters were bound
+                // (before awaiting ExecuteSQL). If the caller mutates the entity in memory while an async
+                // save is in flight (e.g. BaseEntitySaveQueue pipelining an INSERT + UPDATE), merging
+                // post-await entity.GetAll() would falsely mark those in-flight mutations as clean/saved.
+                const preSaveValues = effectiveReturnMode === 'datestamps_only'
+                    ? entity.GetAll()
+                    : undefined;
+                if (preSaveValues && bNewRecord) {
+                    // On a new record saved with 'datestamps_only', the SP does not query BaseView,
+                    // so virtual non-PK fields (e.g. joined name columns) that were never set stay
+                    // null. Omit them from preSaveValues so finalizeSave marks them NotLoaded=true.
+                    for (const f of entity.EntityInfo.Fields) {
+                        if (f.IsVirtual && !f.IsPrimaryKey && (preSaveValues[f.Name] === null || preSaveValues[f.Name] === undefined)) {
+                            delete preSaveValues[f.Name];
+                        }
+                    }
+                }
                 if (entity.TransactionGroup && !bReplay) {
                     // ---- Transaction Group path ----
                     entity.RaiseReadyForTransaction();
                     suspendRefresh();
                     const extraData = this.GetTransactionExtraData(entity);
-                    if (sqlDetails.simpleSQL) {
-                        extraData.simpleSQLFallback = sqlDetails.simpleSQL;
-                    }
+                    Object.defineProperty(extraData, 'simpleSQLFallback', {
+                        get: () => sqlDetails.simpleSQL,
+                        enumerable: true,
+                        configurable: true,
+                    });
                     extraData.entityName = entity.EntityInfo.Name;
-                    entity.TransactionGroup.AddTransaction(new TransactionItem(entity, entityResult.Type === 'create' ? 'Create' : 'Update', sqlDetails.fullSQL, sqlDetails.parameters ?? null, extraData, (transactionResult, success) => {
+                    entity.TransactionGroup.AddTransaction(new TransactionItem(entity, entityResult.Type === 'create' ? 'Create' : 'Update', sqlDetails.literalSQL ?? sqlDetails.fullSQL, sqlDetails.literalSQL ? null : (sqlDetails.parameters ?? null), extraData, (transactionResult, success) => {
                         resumeRefresh();
                         entityResult.EndedAt = new Date();
                         if (success && transactionResult) {
                             this.OnAfterSaveExecute(entity, user, options, saveContext);
                             entityResult.Success = true;
-                            entityResult.NewValues = this.MapTransactionResultToNewValues(transactionResult);
+                            const mergedResult = preSaveValues
+                                ? { ...preSaveValues, ...transactionResult }
+                                : transactionResult;
+                            entityResult.NewValues = this.MapTransactionResultToNewValues(mergedResult);
                         }
                         else {
                             entityResult.Success = false;
@@ -1422,20 +1525,36 @@ export class DatabaseProviderBase extends ProviderBase {
                     if (result && result.length > 0) {
                         this.OnAfterSaveExecute(entity, user, options, saveContext);
                         entityResult.Success = true;
+                        // In 'datestamps_only' mode, the SP/query returned only the PK and
+                        // __mj_CreatedAt/__mj_UpdatedAt columns. Merge those returned values
+                        // onto the pre-save snapshot so caller-set fields remain intact without
+                        // falsely marking concurrent in-flight mutations as clean.
+                        let mergedRow;
+                        if (preSaveValues) {
+                            mergedRow = { ...preSaveValues, ...result[0] };
+                            Object.defineProperty(mergedRow, '__mj_preMerged', {
+                                value: true,
+                                enumerable: false,
+                                configurable: true,
+                            });
+                        }
+                        else {
+                            mergedRow = result[0];
+                        }
                         // OnSaveCompleted may return patch fields (e.g., updated lat/lng
                         // from geocoding) that need to be merged onto the SP result before
                         // the entity loads the data via finalizeSave().
                         const patches = await this.OnSaveCompleted(entity, sqlDetails, user, options, saveContext);
                         if (patches) {
-                            Object.assign(result[0], patches);
+                            Object.assign(mergedRow, patches);
                         }
-                        return result[0];
+                        return mergedRow;
                     }
                     else {
                         if (bNewRecord)
-                            throw new Error(`SQL Error: Error creating new record, no rows returned from SQL: ${sqlDetails.fullSQL}`);
+                            throw new Error(`SQL Error: Error creating new record, no rows returned from SQL: ${sqlDetails.literalSQL ?? sqlDetails.fullSQL}`);
                         else
-                            throw new Error(`SQL Error: Error updating record, no MATCHING rows found within the database: ${sqlDetails.fullSQL}`);
+                            throw new Error(`SQL Error: Error updating record, no MATCHING rows found within the database: ${sqlDetails.literalSQL ?? sqlDetails.fullSQL}`);
                     }
                 }
             }
@@ -1505,20 +1624,24 @@ export class DatabaseProviderBase extends ProviderBase {
                 // ---- Transaction Group path ----
                 entity.RaiseReadyForTransaction();
                 const extraData = this.GetTransactionExtraData(entity);
-                if (entity.EntityInfo.TrackRecordChanges && sqlDetails.simpleSQL) {
-                    extraData.simpleSQLFallback = sqlDetails.simpleSQL;
+                if (entity.EntityInfo.TrackRecordChanges) {
+                    Object.defineProperty(extraData, 'simpleSQLFallback', {
+                        get: () => sqlDetails.simpleSQL,
+                        enumerable: true,
+                        configurable: true,
+                    });
                 }
                 extraData.entityName = entity.EntityInfo.Name;
-                entity.TransactionGroup.AddTransaction(new TransactionItem(entity, 'Delete', sqlDetails.fullSQL, sqlDetails.parameters ?? null, extraData, (transactionResult, success) => {
+                entity.TransactionGroup.AddTransaction(new TransactionItem(entity, 'Delete', sqlDetails.literalSQL ?? sqlDetails.fullSQL, sqlDetails.literalSQL ? null : (sqlDetails.parameters ?? null), extraData, (transactionResult, success) => {
                     entityResult.EndedAt = new Date();
                     if (success && transactionResult) {
-                        this.OnAfterDeleteExecute(entity, user, options);
-                        for (const key of entity.PrimaryKeys) {
-                            if (key.Value !== transactionResult[key.Name]) {
-                                entityResult.Success = false;
-                                entityResult.Message = 'Transaction failed to commit';
-                            }
+                        // Same key check as the direct path, through the provider's own override
+                        // (PostgreSQL accepts its legacy `_result_id` shape there).
+                        if (!this.ValidateDeleteResult(entity, [transactionResult], entityResult)) {
+                            entityResult.Success = false;
+                            return;
                         }
+                        this.OnAfterDeleteExecute(entity, user, options);
                         entityResult.NewValues = this.MapTransactionResultToNewValues(transactionResult);
                         entityResult.Success = true;
                     }