@memberjunction/server 6.2.0-edge.4 → 6.2.0-edge.5

package/src/generic/ResolverBase.ts

@@ -26,7 +26,7 @@ import {
 } from '@memberjunction/core';
 import { MJAuditLogEntity, MJErrorLogEntity, MJUserViewEntityExtended } from '@memberjunction/core-entities';
 import { SQLServerDataProvider } from '@memberjunction/sqlserver-dataprovider';
-import { UserCache } from '@memberjunction/generic-database-provider';
+import { ClientClauseScreen, UserCache, type EntityBaseViewAllowList } from '@memberjunction/generic-database-provider';
 import { PubSubEngine, AuthorizationError } from 'type-graphql';
 import { GraphQLError } from 'graphql';
 import { RefusalExtensions } from './refusalExtensions.js';
@@ -38,8 +38,8 @@ import { RunViewGenericParams, UserPayload } from '../types.js';
 import { RunDynamicViewInput, RunViewByIDInput, RunViewByNameInput } from './RunViewResolver.js';
 import { DeleteOptionsInput } from './DeleteOptionsInput.js';
 import { MJEvent, MJEventType, MJGlobal, ENCRYPTED_SENTINEL, ReplaceByteArraysWithBase64, EscapeSQLString, IsValueEncrypted, IsOnlyTimezoneShift } from '@memberjunction/global';
-import { SQLParser } from '@memberjunction/sql-parser';
-import { PostgreSQLDialect, SQLServerDialect, type SQLParserDialect } from '@memberjunction/sql-dialect';
+import { SQLParser, IsBalancedSQLFragment, LexSQL } from '@memberjunction/sql-parser';
+import type { SQLParserDialect } from '@memberjunction/sql-dialect';
 import { EncryptionEngine, IsEncryptionKeyUnavailable, ReportEncryptionKeyUnavailable } from '@memberjunction/encryption';
 import { PUSH_STATUS_UPDATES_TOPIC, PublishStatusUpdate } from './PushStatusResolver.js';
 import { CACHE_INVALIDATION_TOPIC, MayBroadcastRecordData } from './CacheInvalidationResolver.js';
@@ -47,6 +47,17 @@ import { PubSubManager } from './PubSubManager.js';
 import { FieldMapper } from '@memberjunction/graphql-dataprovider';
 import { Subscription } from 'rxjs';
 
+/**
+ * Entities whose update is always hydrated from the stored row, never from the client's `OldValues___`, though they
+ * do not track record changes: their server-side write rules compare against the prior state, which a client could
+ * otherwise supply itself (lower-case names). See `ResolverBase.MustLoadTruthFromDatabase`.
+ */
+const ENTITIES_THAT_ALWAYS_LOAD_STORED_ROW: ReadonlySet<string> = new Set([
+  'mj: record changes',
+  'mj: conversation details',
+  'mj: data context items',
+]);
+
 export class ResolverBase {
   private static _emit = process.env.CLOUDEVENTS_HTTP_TRANSPORT ? emitterFor(httpTransport(process.env.CLOUDEVENTS_HTTP_TRANSPORT)) : null;
   private static _cloudeventsHeaders = process.env.CLOUDEVENTS_HTTP_HEADERS ? JSON.parse(process.env.CLOUDEVENTS_HTTP_HEADERS) : {};
@@ -86,141 +97,158 @@ export class ResolverBase {
    * @param contextUser - Optional user context for decryption (required for encrypted fields)
    * @returns A new object in transport shape, or null when there is nothing to map
    */
-  protected async MapFieldNamesToCodeNames(
+  /**
+   * Precomputed per-entity / per-request field-mapping plan shared across rows in
+   * {@link MapFieldNamesToCodeNames} and {@link ArrayMapFieldNamesToCodeNames}.
+   */
+  private buildFieldMappingPlan(
     entityName: string,
-    dataObject: any,
     contextUser?: UserInfo,
     provider?: IMetadataProvider,
     deniedReadFields?: Set<string>,
     recordChangeProjector?: RecordChangeFieldSecurityProjector
-  ): Promise<any> {
-    // Return null for empty objects (e.g. when no rows found due to RLS filtering)
+  ): {
+    entityInfo: EntityInfo;
+    fieldsToProcess: Array<{ name: string; mappedFieldName: string; isDenied: boolean }>;
+    readableFields: string[] | null;
+    projector: RecordChangeFieldSecurityProjector | undefined;
+    encryptedFields: EntityFieldInfo[];
+    checkBinary: boolean;
+  } {
+    const md = provider ?? new Metadata();
+    const entityInfo = md.EntityByName(entityName);
+    if (!entityInfo) throw new Error(`Entity ${entityName} not found in metadata`);
+
+    const denied = deniedReadFields ?? (
+      entityInfo.EnableFieldLevelSecurity && contextUser
+        ? entityInfo.GetDeniedReadFields(contextUser)
+        : null
+    );
+
+    const mapper = new FieldMapper();
+    const fieldsToProcess: Array<{ name: string; mappedFieldName: string; isDenied: boolean }> = [];
+    for (const f of entityInfo.Fields) {
+      const mappedFieldName = mapper.MapFieldName(f.CodeName);
+      const isDenied = Boolean(denied?.has(f.Name.trim().toLowerCase()));
+      if (isDenied || mappedFieldName !== f.Name) {
+        fieldsToProcess.push({ name: f.Name, mappedFieldName, isDenied });
+      }
+    }
+
+    const readableFields = this.BuildReadableFieldsTransportValue(entityInfo, denied);
+    const projector = contextUser && RecordChangeFieldSecurityProjector.IsRecordChangesEntity(entityInfo)
+      ? (recordChangeProjector ?? new RecordChangeFieldSecurityProjector(md, contextUser))
+      : undefined;
+    const encryptedFields = entityInfo.EncryptedFields ?? [];
+    const checkBinary = entityInfo.HasBinaryFields !== false;
+
+    return {
+      entityInfo,
+      fieldsToProcess,
+      readableFields,
+      projector,
+      encryptedFields,
+      checkBinary,
+    };
+  }
+
+  /**
+   * Applies synchronous field renaming, FLS stripping, readable-fields transport stamping, and
+   * Record Changes payload projection for a single row using a precomputed mapping plan.
+   */
+  private mapRowSynchronously(
+    dataObject: Record<string, unknown> | null | undefined,
+    plan: ReturnType<ResolverBase['buildFieldMappingPlan']>
+  ): Record<string, unknown> | null {
     if (!dataObject || Object.keys(dataObject).length === 0) {
       return null;
     }
-    // Shallow copy up front so every write below lands on our object, never the caller's.
-    // Binary values are base64 strings everywhere above the providers; any byte array that still
-    // reaches a resolver (custom code, an external driver) is converted here, because GraphQL's
-    // String scalar cannot serialize a Buffer and would fail the whole response.
-    dataObject = ReplaceByteArraysWithBase64({ ...dataObject });
-
-    // for the given entity name provided, check to see if there are any fields
-    // where the code name is different from the field name, and for just those
-    // fields, iterate through the dataObject and REPLACE the property that has the field name
-    // with the CodeName, because we can't transfer those via GraphQL as they are not
-    // valid property names in GraphQL
-    {
-      const md = provider ?? new Metadata();
-      const entityInfo = md.EntityByName(entityName);
-      if (!entityInfo) throw new Error(`Entity ${entityName} not found in metadata`);
-      // const fields = entityInfo.Fields.filter((f) => f.Name !== f.CodeName || f.Name.startsWith('__mj_'));
-      // FIELD-LEVEL SECURITY — this is the authoritative read boundary for every GraphQL
-      // return path (single-record resolvers, external-data-source loads, and anything else
-      // routed through here), sitting exactly where encryption masking already lives.
-      //
-      // The denied set is normally computed ONCE per (entity, user) by the caller and passed
-      // in: this method runs once per ROW, so resolving it here would cost fields x rows
-      // aggregations. We fall back to computing it when a caller doesn't supply it (the
-      // single-record resolvers, where "once per row" and "once per request" are the same
-      // thing) rather than failing open — a missing argument must never mean missing security.
-      const denied = deniedReadFields ?? (
-        entityInfo.EnableFieldLevelSecurity && contextUser
-          ? entityInfo.GetDeniedReadFields(contextUser)
-          : null
-      );
+    let row: Record<string, unknown> = plan.checkBinary
+      ? ReplaceByteArraysWithBase64({ ...dataObject })
+      : { ...dataObject };
 
-      const mapper = new FieldMapper();
-      entityInfo.Fields.forEach((f) => {
-        if (dataObject.hasOwnProperty(f.Name)) {
-          // GraphQL doesn't allow us to pass back fields with __ so we are mapping our special field cases that start with __mj_ to _mj__ for transport - they are converted back on the other side automatically
-          const mappedFieldName = mapper.MapFieldName(f.CodeName);
-          if (denied?.has(f.Name.trim().toLowerCase())) {
-            // Omit entirely rather than nulling — a null is indistinguishable from a real
-            // null value, and the client should see the field as absent, not as empty.
-            // Both key shapes are removed because callers reach this method with rows keyed
-            // either way depending on whether mapping has already run.
-            delete dataObject[f.Name];
-            delete dataObject[mappedFieldName];
-            return;
-          }
-          if (mappedFieldName !== f.Name) {
-            dataObject[mappedFieldName] = dataObject[f.Name];
-            delete dataObject[f.Name];
-          }
+    for (let i = 0; i < plan.fieldsToProcess.length; i++) {
+      const fp = plan.fieldsToProcess[i];
+      if (Object.prototype.hasOwnProperty.call(row, fp.name)) {
+        if (fp.isDenied) {
+          delete row[fp.name];
+          delete row[fp.mappedFieldName];
+        } else {
+          row[fp.mappedFieldName] = row[fp.name];
+          delete row[fp.name];
         }
-      });
-
-      // Field security, second half: say IN-BAND which fields this caller may read. Deleting the
-      // key above is not enough on its own — GraphQL emits every SELECTED field regardless, so a
-      // denied field the client asked for arrives as an explicit null it cannot tell apart from a
-      // genuine one. The client must not settle that from its own metadata: during the window
-      // after a permission change its copy disagrees with ours, and once metadata filtering lands
-      // (issue #3485) it may not hold the permission rows at all. Null for unrestricted callers,
-      // which is nearly every request.
-      const readableFields = this.BuildReadableFieldsTransportValue(entityInfo, denied);
-      if (readableFields) {
-        dataObject[ReadableFieldsTransportKey] = readableFields;
       }
+    }
 
-      // Field security, third part — the audit trail. A 'MJ: Record Changes' row carries the old
-      // and new values of ANOTHER entity's fields, and its own EnableFieldLevelSecurity flag is
-      // off, so everything above is a no-op for it and the payload would flow out untouched. The
-      // projector recomputes the denied set against the entity each row is about; see
-      // RecordChangeFieldSecurityProjector for what each payload column gets.
-      //
-      // Same "a missing argument must never mean missing security" rule as the denied set above:
-      // the projector is normally built ONCE per request by the array caller and passed in, and
-      // we build one here when it wasn't — the single-record resolvers, where once-per-row and
-      // once-per-request are the same thing.
-      if (contextUser && RecordChangeFieldSecurityProjector.IsRecordChangesEntity(entityInfo)) {
-        const projector = recordChangeProjector ?? new RecordChangeFieldSecurityProjector(md, contextUser);
-        dataObject = projector.ProjectRow(dataObject);
-      }
+    if (plan.readableFields) {
+      row[ReadableFieldsTransportKey] = plan.readableFields;
+    }
+
+    if (plan.projector) {
+      row = plan.projector.ProjectRow(row);
+    }
 
-      // Handle encrypted fields - data from raw SQL queries is still encrypted
-      const encryptedFields = entityInfo.EncryptedFields;
-      if (encryptedFields.length > 0) {
-        for (const field of encryptedFields) {
-          const fieldName = field.CodeName;
-          const value = dataObject[fieldName];
-
-          // Skip null/undefined values
-          if (value === null || value === undefined) continue;
-
-          // Check if value is encrypted (raw SQL returns encrypted values)
-          const engine = EncryptionEngine.Instance;
-          await engine.Config(false, contextUser);
-          const keyMarker = field.EncryptionKeyID ? engine.GetKeyByID(field.EncryptionKeyID)?.Marker : undefined;
-          if (typeof value === 'string' && IsValueEncrypted(value, keyMarker)) {
-            if (field.AllowDecryptInAPI) {
-              // Decrypt the value for the client
-              if (contextUser) {
-                try {
-                  const decryptedValue = await engine.Decrypt(value, contextUser);
-                  dataObject[fieldName] = decryptedValue;
-                } catch (err) {
-                  if (IsEncryptionKeyUnavailable(err)) {
-                    ReportEncryptionKeyUnavailable(err);
-                  } else {
-                    LogError(`Failed to decrypt field ${fieldName} for API response: ${err}`);
-                  }
-                  // On decryption failure, use sentinel for safety
-                  dataObject[fieldName] = ENCRYPTED_SENTINEL;
-                }
+    return row;
+  }
+
+  /**
+   * Decrypts or masks encrypted fields on a mapped row when `plan.encryptedFields.length > 0`.
+   */
+  private async decryptRowEncryptedFields(
+    row: Record<string, unknown>,
+    encryptedFields: EntityFieldInfo[],
+    contextUser?: UserInfo
+  ): Promise<void> {
+    for (const field of encryptedFields) {
+      const fieldName = field.CodeName;
+      const value = row[fieldName];
+
+      if (value === null || value === undefined) continue;
+
+      const engine = EncryptionEngine.Instance;
+      await engine.Config(false, contextUser);
+      const keyMarker = field.EncryptionKeyID ? engine.GetKeyByID(field.EncryptionKeyID)?.Marker : undefined;
+      if (typeof value === 'string' && IsValueEncrypted(value, keyMarker)) {
+        if (field.AllowDecryptInAPI) {
+          if (contextUser) {
+            try {
+              const decryptedValue = await engine.Decrypt(value, contextUser);
+              row[fieldName] = decryptedValue;
+            } catch (err) {
+              if (IsEncryptionKeyUnavailable(err)) {
+                ReportEncryptionKeyUnavailable(err);
               } else {
-                // No context user, can't decrypt - use sentinel
-                dataObject[fieldName] = ENCRYPTED_SENTINEL;
+                LogError(`Failed to decrypt field ${fieldName} for API response: ${err}`);
               }
-            } else if (!field.SendEncryptedValue) {
-              // AllowDecryptInAPI=false and SendEncryptedValue=false - use sentinel
-              dataObject[fieldName] = ENCRYPTED_SENTINEL;
+              row[fieldName] = ENCRYPTED_SENTINEL;
             }
-            // else: AllowDecryptInAPI=false and SendEncryptedValue=true - keep encrypted value as-is
+          } else {
+            row[fieldName] = ENCRYPTED_SENTINEL;
           }
+        } else if (!field.SendEncryptedValue) {
+          row[fieldName] = ENCRYPTED_SENTINEL;
         }
       }
     }
-    return dataObject;
+  }
+
+  protected async MapFieldNamesToCodeNames(
+    entityName: string,
+    dataObject: any,
+    contextUser?: UserInfo,
+    provider?: IMetadataProvider,
+    deniedReadFields?: Set<string>,
+    recordChangeProjector?: RecordChangeFieldSecurityProjector
+  ): Promise<any> {
+    if (!dataObject || Object.keys(dataObject).length === 0) {
+      return null;
+    }
+    const plan = this.buildFieldMappingPlan(entityName, contextUser, provider, deniedReadFields, recordChangeProjector);
+    const mapped = this.mapRowSynchronously(dataObject as Record<string, unknown>, plan);
+    if (mapped && plan.encryptedFields.length > 0) {
+      await this.decryptRowEncryptedFields(mapped, plan.encryptedFields, contextUser);
+    }
+    return mapped;
   }
 
   /**
@@ -280,26 +308,17 @@ export class ResolverBase {
     if (!dataObjectArray || dataObjectArray.length === 0) {
       return dataObjectArray;
     }
-    // Resolve the field-security denied set ONCE for the whole array and pass it into the
-    // per-row mapper. MapFieldNamesToCodeNames runs once per row, so computing this inside
-    // the loop would be fields x rows aggregations — 40,000 for a 1,000-row x 40-column
-    // result, each re-scanning the user's roles and allocating. Gated on the entity-level
-    // flag so non-FLS entities (nearly all of them) don't even resolve the entity twice.
-    const md = provider ?? new Metadata();
-    const entityInfo = md.EntityByName(entityName);
-    const deniedReadFields = entityInfo?.EnableFieldLevelSecurity && contextUser
-      ? entityInfo.GetDeniedReadFields(contextUser)
-      : undefined;
-    // Record Changes payload security is resolved per DISTINCT target entity, and one array can
-    // span many — so the projector, which owns that memo, must outlive the row loop. Built once
-    // here for the same reason the denied set above is: GetDeniedReadFields walks every field on
-    // an entity and must never be called per row.
-    const recordChangeProjector = contextUser && RecordChangeFieldSecurityProjector.IsRecordChangesEntity(entityInfo)
-      ? new RecordChangeFieldSecurityProjector(md, contextUser)
-      : undefined;
-    const mapped: any[] = [];
-    for (const element of dataObjectArray) {
-      mapped.push(await this.MapFieldNamesToCodeNames(entityName, element, contextUser, provider, deniedReadFields, recordChangeProjector));
+    // Precompute the entity lookup, FieldMapper plan, denied-fields set, readable-fields list,
+    // and Record Changes projector ONCE for the whole array rather than per row.
+    const plan = this.buildFieldMappingPlan(entityName, contextUser, provider);
+    const hasEncryptedFields = plan.encryptedFields.length > 0;
+    const mapped: Array<Record<string, unknown> | null> = new Array(dataObjectArray.length);
+    for (let i = 0; i < dataObjectArray.length; i++) {
+      const row = this.mapRowSynchronously(dataObjectArray[i] as Record<string, unknown>, plan);
+      if (row && hasEncryptedFields) {
+        await this.decryptRowEncryptedFields(row, plan.encryptedFields, contextUser);
+      }
+      mapped[i] = row;
     }
     return mapped;
   }
@@ -366,13 +385,16 @@ export class ResolverBase {
    * @param dataObject - The data object containing field values
    * @param encryptionEngine - Optional encryption engine for re-encryption (lazy loaded if needed)
    * @param contextUser - User context for encryption operations
+   * @param fieldKey - What the object's keys are: field CodeNames (GraphQL transport rows, the
+   *   default) or field Names (raw provider rows, such as dataset item results)
    * @returns The filtered data object
    */
   protected async FilterEncryptedFieldsForAPI(
     entityName: string,
     dataObject: Record<string, unknown>,
     contextUser: UserInfo,
-    provider?: IMetadataProvider
+    provider?: IMetadataProvider,
+    fieldKey: 'CodeName' | 'Name' = 'CodeName'
   ): Promise<Record<string, unknown>> {
     if (!dataObject) return dataObject;
 
@@ -386,7 +408,7 @@ export class ResolverBase {
 
     // Process each encrypted field
     for (const field of encryptedFields) {
-      const fieldName = field.CodeName;
+      const fieldName = fieldKey === 'Name' ? field.Name : field.CodeName;
       const value = dataObject[fieldName];
 
       // Skip null/undefined values
@@ -427,13 +449,15 @@ export class ResolverBase {
   }
 
   /**
-   * Filters encrypted fields for an array of data objects
+   * Filters encrypted fields for an array of data objects. `fieldKey` is as for
+   * {@link FilterEncryptedFieldsForAPI}.
    */
   protected async ArrayFilterEncryptedFieldsForAPI(
     entityName: string,
     dataObjectArray: Record<string, unknown>[],
     contextUser: UserInfo,
-    provider?: IMetadataProvider
+    provider?: IMetadataProvider,
+    fieldKey: 'CodeName' | 'Name' = 'CodeName'
   ): Promise<Record<string, unknown>[]> {
     if (!dataObjectArray || dataObjectArray.length === 0) return dataObjectArray;
 
@@ -447,7 +471,7 @@ export class ResolverBase {
 
     // Process each element
     for (const element of dataObjectArray) {
-      await this.FilterEncryptedFieldsForAPI(entityName, element, contextUser, provider);
+      await this.FilterEncryptedFieldsForAPI(entityName, element, contextUser, provider, fieldKey);
     }
 
     return dataObjectArray;
@@ -925,11 +949,26 @@ export class ResolverBase {
    * This screen uses `@memberjunction/sql-parser` (same wrap as EDS `assertReadOnlyClause`):
    * wrap the fragment as a single SELECT, fail closed if it does not parse as a read, then
    * allow a FROM only when it is an entity **BaseView**. Base tables (`Meeting`, `__mj.User`)
-   * and catalogs are rejected. Server-internal RunView callers never hit this.
+   * and catalogs are rejected. Server-internal RunView callers get only the provider's
+   * statement-separator check, which cannot refuse a legitimate filter.
+   *
+   * The screen is `ClientClauseScreen` in `@memberjunction/generic-database-provider`, shared
+   * with the REST view routes, ad-hoc list sources, and the provider's saved-view checks.
+   *
+   * Before that parse, the fragment must be balanced on its own (`IsBalancedSQLFragment`). The
+   * provider and the tenant hook splice it inside parentheses and AND their own filters after
+   * it, so a fragment that closes those parentheses (`1=1) OR (1=1`) would leave row-level
+   * security, API-key row filters and tenant scope on one side of an OR the caller controls.
+   * The parse wraps the fragment in parentheses too, which is why it cannot catch this alone.
+   * A backslash inside a quoted string or identifier is refused as well: the parser reads `\'`
+   * inside a plain literal as an escaped quote, but SQL Server and PostgreSQL end the literal
+   * there, so the text after it could be code that the parse-based checks never saw.
    *
    * RLS is applied by RunView as an outer WHERE around the entity being queried, not compiled
-   * into the view. Subqueries against another entity's BaseView therefore do not inherit that
-   * entity's RLS; they are still restricted to the view (not the table).
+   * into the view, and RunView removes denied fields from its own result only. A subquery reads a
+   * base view directly with neither, so the screen refuses one over any entity the user reads under
+   * a row filter or with denied fields (`ClientClauseScreen.AssertUnscopedRead`), the queried
+   * entity included.
    */
   protected assertClientClauseUsesEntityBaseViews(
     clause: string | undefined | null,
@@ -940,43 +979,37 @@ export class ResolverBase {
   ): void {
     if (!clause?.trim()) return;
 
+    // The balance and backslash checks run before the shared screen: it parses the fragment
+    // inside its own parentheses, so it cannot see a fragment that closes the parentheses the
+    // provider and the tenant hook splice it into.
     const dialect = this.dialectForProvider(provider);
     if (SQLParser.HasStackedStatements(clause, dialect)) {
       throw new Error(`Invalid ${label}: multiple statements are not permitted in client-supplied filters`);
     }
-
-    const wrapped =
-      label === 'OrderBy'
-        ? `SELECT 1 FROM __mj_clause_screen ORDER BY ${clause}`
-        : `SELECT 1 FROM __mj_clause_screen WHERE (${clause})`;
-    const parser = new SQLParser(wrapped, dialect);
-    // A clause the provider checks again on its own (an aggregate expression) leaves text the parser
-    // cannot read to that per-item check instead of failing the whole request.
-    if (!parser.IsValid && options?.LeaveUnparseableToProvider) return;
-    if (!parser.IsValid || parser.HasWriteStatement || parser.StatementKind !== 'select') {
-      throw new Error(
-        `Invalid ${label}: not a safe read-only filter fragment — refusing under uncertainty`,
-      );
+    const balance = IsBalancedSQLFragment(clause, dialect);
+    if (!balance.IsBalanced) {
+      throw new Error(`Invalid ${label}: ${balance.Reason}`);
     }
-    if (this.astContainsWriteNode(parser.AST)) {
-      throw new Error(`Invalid ${label}: write/DDL nested in a subquery is not permitted`);
+    if (this.hasBackslashInQuotes(clause, dialect)) {
+      throw new Error(`Invalid ${label}: a backslash inside a quoted string or identifier is not permitted in client-supplied filters`);
     }
 
-    const allowed = this.entityBaseViewAllowList(provider);
-    const tables = SQLParser.ExtractTableRefs(wrapped, dialect);
-    for (const t of tables) {
-      const table = this.stripSqlIdent(t.TableName);
-      const schema = this.stripSqlIdent(t.SchemaName);
-      if (table.toLowerCase() === '__mj_clause_screen') continue;
-      this.assertTableRefReadable(allowed, schema, table, label, user);
-    }
+    ClientClauseScreen.AssertClauseUsesEntityBaseViews(clause, label, provider?.Entities ?? [], dialect, user, options);
   }
 
   /**
    * SECURITY — screen for a FULL client-supplied SQL statement (ad-hoc query path). In addition
    * to the SELECT-only validation the caller performs, every table reference must resolve to an
-   * entity BaseView the acting user can read UNSCOPED. CTE names defined by the statement itself
-   * are excluded from the check. Fails closed: no user, or an unresolvable reference, refuses.
+   * entity BaseView the acting user can read UNSCOPED. A reference is excluded only when it reads a
+   * CTE that is in scope there (`StatementRelation.IsCTEReference`): unqualified, declared by its
+   * own SELECT or an enclosing one, and matched by the dialect's identifier rules. A name reserved
+   * for the SQL parser is refused before parsing (`ClientClauseScreen.AssertNoIdentifierAliasNames`).
+   * Fails closed: no user, or an unresolvable reference, refuses.
+   *
+   * The references come from a walk of the whole parse tree
+   * (`ClientClauseScreen.CollectStatementReads`). The statement is refused when that walk does not
+   * account for every SELECT in the text, or when a FROM item names no table (a table function or
+   * a row list).
    *
    * Entity CanRead is not unscoped base-view read authority: RunView narrows a granted read with
    * row-level security and denied-field projection, and raw SQL applies neither. Rather than try
@@ -994,6 +1027,7 @@ export class ResolverBase {
       throw new Error(`Invalid ${label}: no acting user resolved for ad-hoc SQL — refusing`);
     }
     const dialect = this.dialectForProvider(provider);
+    ClientClauseScreen.AssertNoIdentifierAliasNames(sqlText, dialect, label);
     const parser = new SQLParser(sqlText, dialect);
     if (!parser.IsValid || parser.HasWriteStatement || parser.StatementKind !== 'select') {
       throw new Error(`Invalid ${label}: not a safe read-only statement — refusing under uncertainty`);
@@ -1001,52 +1035,19 @@ export class ResolverBase {
     if (this.astContainsWriteNode(parser.AST)) {
       throw new Error(`Invalid ${label}: write/DDL nested in a subquery is not permitted`);
     }
-    const cteNames = new Set<string>();
-    this.collectCTENames(parser.AST, cteNames);
+    const reads = ClientClauseScreen.CollectStatementReads(parser);
+    ClientClauseScreen.AssertEverySelectAccountedFor(sqlText, dialect, reads, label);
     const allowed = this.entityBaseViewAllowList(provider);
-    const tables = SQLParser.ExtractTableRefs(sqlText, dialect);
-    for (const t of tables) {
-      const table = this.stripSqlIdent(t.TableName);
-      const schema = this.stripSqlIdent(t.SchemaName);
-      if (this.isUnqualifiedSchema(schema) && cteNames.has(table.toLowerCase())) continue;
-      const entity = this.assertTableRefReadable(allowed, schema, table, label, user);
-      this.assertUnscopedRead(entity, user, label);
+    for (const relation of reads.Relations) {
+      if (relation.IsCTEReference) continue;
+      const entity = ClientClauseScreen.AssertRelationReadable(allowed, relation, label, user);
+      if (entity) this.assertUnscopedRead(entity, user, label);
     }
   }
 
-  /** Refuses an entity whose read is narrowed for this user by row-level security or field-level denials. */
+  /** Refuses an entity whose read is narrowed for this user; see `ClientClauseScreen.AssertUnscopedRead`. */
   private assertUnscopedRead(entity: EntityInfo, user: UserInfo, label: string): void {
-    if (entity.GetEffectiveRowFilterWhereClause(user, EntityPermissionType.Read, '').length > 0) {
-      throw new Error(
-        `Invalid ${label}: entity '${entity.Name}' is row-level-security filtered for you — ad-hoc SQL cannot apply that filter; use RunView`,
-      );
-    }
-    if (entity.GetDeniedReadFields(user).size > 0) {
-      throw new Error(
-        `Invalid ${label}: entity '${entity.Name}' has fields you are not permitted to read — ad-hoc SQL cannot project them away; use RunView`,
-      );
-    }
-  }
-
-  /** Walks a parsed AST collecting the names of CTEs the statement itself defines. */
-  private collectCTENames(node: unknown, names: Set<string>): void {
-    if (!node || typeof node !== 'object') return;
-    if (Array.isArray(node)) {
-      for (const n of node) this.collectCTENames(n, names);
-      return;
-    }
-    const obj = node as Record<string, unknown>;
-    const withClause = obj.with;
-    if (Array.isArray(withClause)) {
-      for (const cte of withClause) {
-        const name = (cte as Record<string, unknown>)?.name;
-        if (typeof name === 'string') names.add(name.toLowerCase());
-        else if (name && typeof name === 'object' && typeof (name as Record<string, unknown>).value === 'string') {
-          names.add(((name as Record<string, unknown>).value as string).toLowerCase());
-        }
-      }
-    }
-    for (const v of Object.values(obj)) this.collectCTENames(v, names);
+    ClientClauseScreen.AssertUnscopedRead(entity, user, label);
   }
 
   /**
@@ -1095,105 +1096,26 @@ export class ResolverBase {
     }
   }
 
-  /** Same write-node walk as EDS `sqlReadOnlyScreen.astContainsWriteNode`. */
-  private astContainsWriteNode(node: unknown): boolean {
-    if (!node || typeof node !== 'object') return false;
-    if (Array.isArray(node)) return node.some((n) => this.astContainsWriteNode(n));
-    const obj = node as Record<string, unknown>;
-    const type = obj.type;
-    if (typeof type === 'string' && ResolverBase.WRITE_NODE_TYPES.has(type.toLowerCase())) {
-      return true;
-    }
-    return Object.values(obj).some((v) => this.astContainsWriteNode(v));
-  }
-
-  private static readonly WRITE_NODE_TYPES = new Set<string>([
-    'insert', 'update', 'delete', 'merge', 'replace', 'drop', 'create', 'alter', 'truncate',
-    'rename', 'call', 'exec', 'execute', 'grant', 'revoke', 'use', 'load', 'copy', 'do',
-  ]);
-
-  private dialectForProvider(provider?: IMetadataProvider): SQLParserDialect {
-    const name = provider?.constructor?.name ?? '';
-    if (/postgres/i.test(name)) return new PostgreSQLDialect();
-    return new SQLServerDialect();
-  }
-
-  /** True for a reference with no schema — which `SQLParser.ExtractTableRefs` reports as `dbo`. */
-  private isUnqualifiedSchema(schema: string): boolean {
-    return !schema || schema.toLowerCase() === 'dbo';
+  /**
+   * True when a string literal or quoted identifier in `clause`, read the way the database reads
+   * it, contains a backslash (see {@link assertClientClauseUsesEntityBaseViews}).
+   */
+  private hasBackslashInQuotes(clause: string, dialect: SQLParserDialect): boolean {
+    return LexSQL(clause, dialect).some((t) => (t.Kind === 'string' || t.Kind === 'identifier') && t.Text.includes('\\'));
   }
 
-  private stripSqlIdent(name: string | null | undefined): string {
-    if (!name) return '';
-    return name.replace(/^\[|\]$/g, '').replace(/^"|"$/g, '').replace(/^`|`$/g, '');
+  /** Write/DDL node walk shared with the clause screen; see `ClientClauseScreen.AstContainsWriteNode`. */
+  private astContainsWriteNode(node: unknown): boolean {
+    return ClientClauseScreen.AstContainsWriteNode(node);
   }
 
-  /**
-   * `bare` maps an unqualified view name to its entity, or to `null` when the name exists in
-   * more than one schema — the database, not this screen, would decide which one runs.
-   */
-  private entityBaseViewAllowList(provider?: IMetadataProvider): {
-    qualified: Map<string, EntityInfo>;
-    bare: Map<string, EntityInfo | null>;
-  } {
-    const qualified = new Map<string, EntityInfo>();
-    const bare = new Map<string, EntityInfo | null>();
-    const entities = provider?.Entities ?? [];
-    for (const e of entities) {
-      const view = this.stripSqlIdent(e.BaseView);
-      if (!view) continue;
-      const schema = this.stripSqlIdent(e.SchemaName);
-      const bareKey = view.toLowerCase();
-      bare.set(bareKey, bare.has(bareKey) ? null : e);
-      if (schema) qualified.set(`${schema}.${view}`.toLowerCase(), e);
-    }
-    return { qualified, bare };
+  private dialectForProvider(provider?: IMetadataProvider): SQLParserDialect {
+    return ClientClauseScreen.DialectFor(provider);
   }
 
-  /**
-   * SECURITY — resolves a table reference against the entity BaseView allow-list and, when an
-   * acting user is supplied, additionally requires that user to hold CanRead on the referenced
-   * entity. Base views do not embed RLS and entity permissions are otherwise checked only on
-   * the TOP entity of a request, so without this check a subquery (or ad-hoc query) could read
-   * entities the caller has no read grant on.
-   *
-   * A schema-qualified reference must match that exact schema — `secret.vwFoo` never resolves
-   * through another schema's `vwFoo`. An unqualified reference resolves only when the view name
-   * is unique across schemas; an ambiguous one must be qualified, because the database's
-   * default-schema resolution, not this screen, would pick which view actually runs.
-   *
-   * `SQLParser.ExtractTableRefs` reports an unqualified reference as schema `dbo`, so `dbo` is
-   * indistinguishable from "no schema" here and both take the unqualified path (an exact
-   * `dbo.<view>` entity wins first).
-   */
-  private assertTableRefReadable(
-    allowed: { qualified: Map<string, EntityInfo>; bare: Map<string, EntityInfo | null> },
-    schema: string,
-    table: string,
-    label: string,
-    user?: UserInfo,
-  ): EntityInfo {
-    const qualified = allowed.qualified.get(`${schema}.${table}`.toLowerCase());
-    const entity = this.isUnqualifiedSchema(schema) ? qualified ?? allowed.bare.get(table.toLowerCase()) : qualified;
-    if (entity === null) {
-      throw new Error(
-        `Invalid ${label}: '${table}' is a base view in more than one schema — qualify it with its schema`,
-      );
-    }
-    if (!entity) {
-      throw new Error(
-        `Invalid ${label}: subquery must use an entity base view, not '${schema ? schema + '.' : ''}${table}'`,
-      );
-    }
-    if (user) {
-      const perms = entity.GetUserPermisions(user);
-      if (!perms.CanRead) {
-        throw new Error(
-          `Invalid ${label}: you do not have read permission on entity '${entity.Name}' referenced by '${schema ? schema + '.' : ''}${table}'`,
-        );
-      }
-    }
-    return entity;
+  /** The entity base-view allow-list for a provider's entities. */
+  private entityBaseViewAllowList(provider?: IMetadataProvider): EntityBaseViewAllowList {
+    return ClientClauseScreen.BuildEntityBaseViewAllowList(provider?.Entities ?? []);
   }
 
   /**
@@ -1913,13 +1835,41 @@ export class ResolverBase {
       });
   }
   
+  /**
+   * Checks whether `oldValues` (`input.OldValues___`) contains every writable in-table field of
+   * `entityInfo` (plus primary keys).
+   *
+   * With the delta-only `Save()` wire protocol, `GraphQLDataProvider.Save()` sends only primary keys
+   * and dirty writable fields in `vars.input` and `OldValues___` on normal updates. Hydrating an
+   * entity directly from a partial-delta `OldValues___` via `entityObject.LoadFromData(oldValues)`
+   * (without first loading the authoritative row via `InnerLoad`) would leave every untouched column
+   * `NotLoaded = true` (or `null`), causing `spUpdate` or custom `Validate()` hooks to see incomplete
+   * state. Returning `false` for partial deltas ensures {@link MustLoadTruthFromDatabase} routes
+   * delta updates through `InnerLoad(cKey)` + {@link TestAndSetClientOldValuesToDBValues}, even on
+   * entities with `TrackRecordChanges === false`.
+   */
+  protected hasCompleteOldValues(
+    entityInfo: EntityInfo,
+    oldValues?: Array<{ Key: string; Value: unknown }> | null
+  ): boolean {
+    if (!oldValues || oldValues.length === 0) {
+      return false;
+    }
+    const providedKeys = new Set(oldValues.map((kv) => kv.Key));
+    return entityInfo.Fields.every(
+      (f) => (!f.IsPrimaryKey && (f.ReadOnly || f.IsVirtual)) || providedKeys.has(f.Name) || providedKeys.has(f.CodeName)
+    );
+  }
+
   /**
    * Whether `UpdateRecord` must hydrate the entity from the DATABASE rather than from the
    * client's `OldValues___`.
    *
-   * The `OldValues___` path exists as an optimization: when nothing needs the true prior state,
-   * the client already holds it and a round trip is wasted. Each condition below is a reason
-   * that assumption fails.
+   * The `OldValues___` path exists as an optimization when a client supplies the complete pre-save
+   * record (e.g. `IgnoreDirtyState: true`) and nothing else requires the database's authoritative prior
+   * state. Under the delta-only `Save()` wire protocol, normal updates send only primary keys and dirty
+   * writable fields in `OldValues___`, so {@link hasCompleteOldValues} returns `false` and forces the
+   * database load (`InnerLoad`) so untouched columns retain their stored values.
    *
    * `EnableFieldLevelSecurity` is the security-critical one, and it is NOT redundant with
    * `hasDeniedReadFields`. That flag reports READ denials, and the canonical FLS configuration is
@@ -1945,6 +1895,16 @@ export class ResolverBase {
    * caller pin forged audit columns as both old and new values. It doesn't track its own changes, so
    * nothing else forces the load.
    *
+   * `MJ: Data Context Items` always loads from the database for the same reason: its server class
+   * lets only an administrator save an item that holds SQL, and decides that from the item's `Type`
+   * and `SQL`. Hydrated from OldValues, a client could leave both out, and the stored procedure would
+   * keep the stored SQL item while the rule saw none.
+   *
+   * `MJ: Conversation Details` always loads for the same reason: its write gate decides from the stored
+   * row (who wrote a person's message, its role, the owner's rating), and an OldValues-hydrated entity
+   * would let a grantee claim to be a message's author while rewriting it. See
+   * {@link ENTITIES_THAT_ALWAYS_LOAD_STORED_ROW}.
+   *
    * A caller under an Update row filter (a role's Update RLS filter or an API-key row filter) always
    * loads from the database too (#4919). The save's post-image check skips its query when no column
    * the filter reads is dirty, and dirtiness measured against client OldValues is the caller's to
@@ -1962,8 +1922,8 @@ export class ResolverBase {
   ): boolean {
     return (
       entityInfo.TrackRecordChanges ||
-      entityInfo.Name.trim().toLowerCase() === 'mj: record changes' ||
-      !input.OldValues___ ||
+      ENTITIES_THAT_ALWAYS_LOAD_STORED_ROW.has(entityInfo.Name.trim().toLowerCase()) ||
+      !this.hasCompleteOldValues(entityInfo, input.OldValues___) ||
       hasDeniedReadFields ||
       hasNarrowedAuditPayload ||
       entityInfo.EnableFieldLevelSecurity ||
@@ -2137,19 +2097,36 @@ export class ResolverBase {
   }
 
   /**
-   * This routine compares the OldValues property in the input object to the values in the DB that we just loaded. If there are differences, we need to check to see if the client
-   * is trying to update any of those fields (e.g. overlap). If there is overlap, we throw an error. If there is no overlap, we can proceed with the update even if the DB Values
-   * and the ClientOldValues are not 100% the same, so long as there is no overlap in the specific FIELDS that are different.
+   * Compares the client-supplied `OldValues___` array in `input` against the authoritative database
+   * values just loaded onto `entityObject` via `InnerLoad`, checks for overlapping concurrent field
+   * modifications, and then applies `clientNewValues` onto `entityObject` via `SetMany`.
    *
-   * ASSUMES: input object has an OldValues___ property that is an array of Key/Value pairs that represent the old values of the record that the client is trying to update.
+   * Supports both full-record and delta-only `OldValues___` payloads: only keys present in
+   * `input.OldValues___` (e.g. primary keys + dirty writable fields) are compared against `dbValues`,
+   * and `entityObject.SetMany(clientNewValues)` mutates only the fields sent in `clientNewValues`
+   * while preserving all untouched DB-loaded fields on `entityObject`.
+   *
+   * ASSUMES: `input` has an `OldValues___` property that is an array of Key/Value pairs representing
+   * the pre-mutation values of the fields the client is updating.
    */
+  /**
+   * Resolves a field on `entityInfo` by `CodeName` or `Name` in $O(1)$ via `FieldByCodeName` /
+   * `FieldByName` when available on `EntityInfo`, falling back to `.Fields.find()` for test stubs.
+   */
+  private findEntityFieldByCodeNameOrName(entityInfo: EntityInfo, key: string): EntityFieldInfo | undefined {
+    if (typeof entityInfo.FieldByCodeName === 'function') {
+      return entityInfo.FieldByCodeName(key) ?? entityInfo.FieldByName(key);
+    }
+    return entityInfo.Fields.find((f) => f.CodeName === key || f.Name === key);
+  }
+
   protected async TestAndSetClientOldValuesToDBValues(input: any, clientNewValues: any, entityObject: BaseEntity, contextUser: UserInfo) {
     // we have OldValues, so we need to compare them to the values we just loaded from the DB
     const clientOldValues = {};
     // for each item in the oldValues array, add it to the clientOldValues object
     input.OldValues___.forEach((item) => {
       // we need to do a quick transform on the values to make sure they match the TS Type for the given field because item.Value will always be a string
-      const field = entityObject.EntityInfo.Fields.find((f) => f.CodeName === item.Key);
+      const field = this.findEntityFieldByCodeNameOrName(entityObject.EntityInfo, item.Key);
       const val = this.ClientOldValueToFieldValue(field, item.Value);
       clientOldValues[item.Key] = val;
     });
@@ -2161,19 +2138,20 @@ export class ResolverBase {
     // now we need to compare clientOldValues and dbValues and have a new array that has entries for any differences and have FieldName, clientOldValue and dbValue as properties
     const dbDifferences = [];
     Object.keys(clientOldValues).forEach((key) => {
-      const f = entityObject.EntityInfo.Fields.find((f) => f.CodeName === key);
+      const f = this.findEntityFieldByCodeNameOrName(entityObject.EntityInfo, key);
+      const dbVal = f && f.Name in dbValues ? dbValues[f.Name] : dbValues[key];
       let different = false;
       switch (typeof clientOldValues[key]) {
         case 'number':
-          different = clientOldValues[key] !== dbValues[key];
+          different = clientOldValues[key] !== dbVal;
           break;
         case 'boolean':
-          different = clientOldValues[key] !== dbValues[key];
+          different = clientOldValues[key] !== dbVal;
           break;
         case 'object':
           if (clientOldValues[key] instanceof Date) {
             const clientDate = clientOldValues[key] as Date;
-            const dbDate = dbValues[key];
+            const dbDate = dbVal;
             if (dbDate == null || !(dbDate instanceof Date)) {
               different = true;
             } else if (clientDate.getTime() !== dbDate.getTime()) {
@@ -2191,11 +2169,11 @@ export class ResolverBase {
               }
             }
           } else if (clientOldValues[key] === null) {
-            different = dbValues[key] !== null;
+            different = dbVal !== null;
           }
           break;
         default:
-          different = clientOldValues[key] !== dbValues[key];
+          different = clientOldValues[key] !== dbVal;
           break;
       }
       if (different && f && !f.ReadOnly) {
@@ -2203,7 +2181,7 @@ export class ResolverBase {
         dbDifferences.push({
           FieldName: key,
           ClientOldValue: clientOldValues[key],
-          DBValue: dbValues[key],
+          DBValue: dbVal,
         });
       }
     });
@@ -2213,8 +2191,8 @@ export class ResolverBase {
       // first step is to get clientNewValues into an object that is like clientOldValues, get the diff and then compare that diff to the differences array that shows diff between DB and ClientOld
       const clientDifferences = [];
       Object.keys(clientOldValues).forEach((key) => {
-        const f = entityObject.EntityInfo.Fields.find((f) => f.CodeName === key);
-        if (clientOldValues[key] !== clientNewValues[key] && f && f.AllowUpdateAPI && !f.IsPrimaryKey) {
+        const f = this.findEntityFieldByCodeNameOrName(entityObject.EntityInfo, key);
+        if (key in clientNewValues && clientOldValues[key] !== clientNewValues[key] && f && f.AllowUpdateAPI && !f.IsPrimaryKey) {
           // only include updateable fields
           clientDifferences.push({
             FieldName: key,
@@ -2226,7 +2204,8 @@ export class ResolverBase {
 
       // now we have clientDifferences which shows what the client thinks they are changing. And, we have the dbDifferences array that shows changes between the clientOldValues and the dbValues
       // if there is ANY overlap in the FIELDS that appear in both arrays, we need to log a warning but allow the save to continue
-      const overlap = clientDifferences.filter((cd) => dbDifferences.find((dd) => dd.FieldName === cd.FieldName));
+      const dbDiffFieldNames = new Set(dbDifferences.map((dd) => dd.FieldName));
+      const overlap = clientDifferences.filter((cd) => dbDiffFieldNames.has(cd.FieldName));
       if (overlap.length > 0) {
         const msg = {
           Message: