@memberjunction/server 6.2.0-edge.2 → 6.2.0-edge.3

package/src/generic/ResolverBase.ts

@@ -37,7 +37,7 @@ import { httpTransport, CloudEvent, emitterFor } from 'cloudevents';
 import { RunViewGenericParams, UserPayload } from '../types.js';
 import { RunDynamicViewInput, RunViewByIDInput, RunViewByNameInput } from './RunViewResolver.js';
 import { DeleteOptionsInput } from './DeleteOptionsInput.js';
-import { MJEvent, MJEventType, MJGlobal, ENCRYPTED_SENTINEL, EscapeSQLString, IsValueEncrypted, IsOnlyTimezoneShift } from '@memberjunction/global';
+import { MJEvent, MJEventType, MJGlobal, ENCRYPTED_SENTINEL, ReplaceByteArraysWithBase64, EscapeSQLString, IsValueEncrypted, IsOnlyTimezoneShift } from '@memberjunction/global';
 import { SQLParser } from '@memberjunction/sql-parser';
 import { PostgreSQLDialect, SQLServerDialect, type SQLParserDialect } from '@memberjunction/sql-dialect';
 import { EncryptionEngine } from '@memberjunction/encryption';
@@ -99,7 +99,10 @@ export class ResolverBase {
       return null;
     }
     // Shallow copy up front so every write below lands on our object, never the caller's.
-    dataObject = { ...dataObject };
+    // Binary values are base64 strings everywhere above the providers; any byte array that still
+    // reaches a resolver (custom code, an external driver) is converted here, because GraphQL's
+    // String scalar cannot serialize a Buffer and would fail the whole response.
+    dataObject = ReplaceByteArraysWithBase64({ ...dataObject });
 
     // for the given entity name provided, check to see if there are any fields
     // where the code name is different from the field name, and for just those
@@ -566,7 +569,8 @@ export class ResolverBase {
             ? CompositeKey.FromKeyValuePairs((viewInput.AfterKey as { KeyValuePairs: { FieldName: string; Value: string }[] }).KeyValuePairs)
             : undefined,
           viewInput.BypassCache,
-          viewInput.DataSource
+          viewInput.DataSource,
+          viewInput.IncludeBinaryFields
         );
       }
       else {
@@ -610,7 +614,8 @@ export class ResolverBase {
         viewInput.Aggregates,
         undefined,
         viewInput.BypassCache,
-        viewInput.DataSource
+        viewInput.DataSource,
+        viewInput.IncludeBinaryFields
       );
     } catch (err) {
       console.log(err);
@@ -657,7 +662,8 @@ export class ResolverBase {
         viewInput.Aggregates,
         undefined,
         viewInput.BypassCache,
-        viewInput.DataSource
+        viewInput.DataSource,
+        viewInput.IncludeBinaryFields
       );
     } catch (err) {
       console.log(err);
@@ -732,6 +738,7 @@ export class ResolverBase {
           aggregates: viewInput.Aggregates,
           bypassCache: viewInput.BypassCache,
           dataSource: viewInput.DataSource,
+          includeBinaryFields: viewInput.IncludeBinaryFields,
         });
       } catch (err) {
         LogError(err);
@@ -924,6 +931,7 @@ export class ResolverBase {
     clause: string | undefined | null,
     label: string,
     provider?: IMetadataProvider,
+    user?: UserInfo,
   ): void {
     if (!clause?.trim()) return;
 
@@ -952,15 +960,87 @@ export class ResolverBase {
       const table = this.stripSqlIdent(t.TableName);
       const schema = this.stripSqlIdent(t.SchemaName);
       if (table.toLowerCase() === '__mj_clause_screen') continue;
-      const qualified = `${schema}.${table}`.toLowerCase();
-      const bare = table.toLowerCase();
-      if (allowed.qualified.has(qualified) || allowed.bare.has(bare)) continue;
+      this.assertTableRefReadable(allowed, schema, table, label, user);
+    }
+  }
+
+  /**
+   * SECURITY — screen for a FULL client-supplied SQL statement (ad-hoc query path). In addition
+   * to the SELECT-only validation the caller performs, every table reference must resolve to an
+   * entity BaseView the acting user can read UNSCOPED. CTE names defined by the statement itself
+   * are excluded from the check. Fails closed: no user, or an unresolvable reference, refuses.
+   *
+   * Entity CanRead is not unscoped base-view read authority: RunView narrows a granted read with
+   * row-level security and denied-field projection, and raw SQL applies neither. Rather than try
+   * to rewrite arbitrary SQL (joins, CTEs) to compose that policy, this refuses any entity that
+   * carries a row filter or a denied field for the caller — those reads belong on RunView. For
+   * the remaining entities CanRead genuinely means "every row and field of the view".
+   */
+  protected assertFullQueryUsesReadableEntityViews(
+    sqlText: string,
+    provider: IMetadataProvider | undefined,
+    user: UserInfo | undefined,
+    label = 'SQL',
+  ): void {
+    if (!user) {
+      throw new Error(`Invalid ${label}: no acting user resolved for ad-hoc SQL — refusing`);
+    }
+    const dialect = this.dialectForProvider(provider);
+    const parser = new SQLParser(sqlText, dialect);
+    if (!parser.IsValid || parser.HasWriteStatement || parser.StatementKind !== 'select') {
+      throw new Error(`Invalid ${label}: not a safe read-only statement — refusing under uncertainty`);
+    }
+    if (this.astContainsWriteNode(parser.AST)) {
+      throw new Error(`Invalid ${label}: write/DDL nested in a subquery is not permitted`);
+    }
+    const cteNames = new Set<string>();
+    this.collectCTENames(parser.AST, cteNames);
+    const allowed = this.entityBaseViewAllowList(provider);
+    const tables = SQLParser.ExtractTableRefs(sqlText, dialect);
+    for (const t of tables) {
+      const table = this.stripSqlIdent(t.TableName);
+      const schema = this.stripSqlIdent(t.SchemaName);
+      if (this.isUnqualifiedSchema(schema) && cteNames.has(table.toLowerCase())) continue;
+      const entity = this.assertTableRefReadable(allowed, schema, table, label, user);
+      this.assertUnscopedRead(entity, user, label);
+    }
+  }
+
+  /** Refuses an entity whose read is narrowed for this user by row-level security or field-level denials. */
+  private assertUnscopedRead(entity: EntityInfo, user: UserInfo, label: string): void {
+    if (entity.GetEffectiveRowFilterWhereClause(user, EntityPermissionType.Read, '').length > 0) {
       throw new Error(
-        `Invalid ${label}: subquery must use an entity base view, not '${schema}.${table}'`,
+        `Invalid ${label}: entity '${entity.Name}' is row-level-security filtered for you — ad-hoc SQL cannot apply that filter; use RunView`,
+      );
+    }
+    if (entity.GetDeniedReadFields(user).size > 0) {
+      throw new Error(
+        `Invalid ${label}: entity '${entity.Name}' has fields you are not permitted to read — ad-hoc SQL cannot project them away; use RunView`,
       );
     }
   }
 
+  /** Walks a parsed AST collecting the names of CTEs the statement itself defines. */
+  private collectCTENames(node: unknown, names: Set<string>): void {
+    if (!node || typeof node !== 'object') return;
+    if (Array.isArray(node)) {
+      for (const n of node) this.collectCTENames(n, names);
+      return;
+    }
+    const obj = node as Record<string, unknown>;
+    const withClause = obj.with;
+    if (Array.isArray(withClause)) {
+      for (const cte of withClause) {
+        const name = (cte as Record<string, unknown>)?.name;
+        if (typeof name === 'string') names.add(name.toLowerCase());
+        else if (name && typeof name === 'object' && typeof (name as Record<string, unknown>).value === 'string') {
+          names.add(((name as Record<string, unknown>).value as string).toLowerCase());
+        }
+      }
+    }
+    for (const v of Object.values(obj)) this.collectCTENames(v, names);
+  }
+
   /**
    * Applies {@link assertClientClauseUsesEntityBaseViews} to every client-supplied clause
    * a view request can carry that is actually a SQL fragment. GraphQL entry points
@@ -991,10 +1071,11 @@ export class ResolverBase {
       overrideExcludeFilter?: string | null;
     },
     provider?: IMetadataProvider,
+    user?: UserInfo,
   ): void {
-    this.assertClientClauseUsesEntityBaseViews(clauses.extraFilter, 'ExtraFilter', provider);
-    this.assertClientClauseUsesEntityBaseViews(clauses.orderBy, 'OrderBy', provider);
-    this.assertClientClauseUsesEntityBaseViews(clauses.overrideExcludeFilter, 'OverrideExcludeFilter', provider);
+    this.assertClientClauseUsesEntityBaseViews(clauses.extraFilter, 'ExtraFilter', provider, user);
+    this.assertClientClauseUsesEntityBaseViews(clauses.orderBy, 'OrderBy', provider, user);
+    this.assertClientClauseUsesEntityBaseViews(clauses.overrideExcludeFilter, 'OverrideExcludeFilter', provider, user);
   }
 
   /** Same write-node walk as EDS `sqlReadOnlyScreen.astContainsWriteNode`. */
@@ -1020,28 +1101,84 @@ export class ResolverBase {
     return new SQLServerDialect();
   }
 
+  /** True for a reference with no schema — which `SQLParser.ExtractTableRefs` reports as `dbo`. */
+  private isUnqualifiedSchema(schema: string): boolean {
+    return !schema || schema.toLowerCase() === 'dbo';
+  }
+
   private stripSqlIdent(name: string | null | undefined): string {
     if (!name) return '';
     return name.replace(/^\[|\]$/g, '').replace(/^"|"$/g, '').replace(/^`|`$/g, '');
   }
 
+  /**
+   * `bare` maps an unqualified view name to its entity, or to `null` when the name exists in
+   * more than one schema — the database, not this screen, would decide which one runs.
+   */
   private entityBaseViewAllowList(provider?: IMetadataProvider): {
-    qualified: Set<string>;
-    bare: Set<string>;
+    qualified: Map<string, EntityInfo>;
+    bare: Map<string, EntityInfo | null>;
   } {
-    const qualified = new Set<string>();
-    const bare = new Set<string>();
+    const qualified = new Map<string, EntityInfo>();
+    const bare = new Map<string, EntityInfo | null>();
     const entities = provider?.Entities ?? [];
     for (const e of entities) {
       const view = this.stripSqlIdent(e.BaseView);
       if (!view) continue;
       const schema = this.stripSqlIdent(e.SchemaName);
-      bare.add(view.toLowerCase());
-      if (schema) qualified.add(`${schema}.${view}`.toLowerCase());
+      const bareKey = view.toLowerCase();
+      bare.set(bareKey, bare.has(bareKey) ? null : e);
+      if (schema) qualified.set(`${schema}.${view}`.toLowerCase(), e);
     }
     return { qualified, bare };
   }
 
+  /**
+   * SECURITY — resolves a table reference against the entity BaseView allow-list and, when an
+   * acting user is supplied, additionally requires that user to hold CanRead on the referenced
+   * entity. Base views do not embed RLS and entity permissions are otherwise checked only on
+   * the TOP entity of a request, so without this check a subquery (or ad-hoc query) could read
+   * entities the caller has no read grant on.
+   *
+   * A schema-qualified reference must match that exact schema — `secret.vwFoo` never resolves
+   * through another schema's `vwFoo`. An unqualified reference resolves only when the view name
+   * is unique across schemas; an ambiguous one must be qualified, because the database's
+   * default-schema resolution, not this screen, would pick which view actually runs.
+   *
+   * `SQLParser.ExtractTableRefs` reports an unqualified reference as schema `dbo`, so `dbo` is
+   * indistinguishable from "no schema" here and both take the unqualified path (an exact
+   * `dbo.<view>` entity wins first).
+   */
+  private assertTableRefReadable(
+    allowed: { qualified: Map<string, EntityInfo>; bare: Map<string, EntityInfo | null> },
+    schema: string,
+    table: string,
+    label: string,
+    user?: UserInfo,
+  ): EntityInfo {
+    const qualified = allowed.qualified.get(`${schema}.${table}`.toLowerCase());
+    const entity = this.isUnqualifiedSchema(schema) ? qualified ?? allowed.bare.get(table.toLowerCase()) : qualified;
+    if (entity === null) {
+      throw new Error(
+        `Invalid ${label}: '${table}' is a base view in more than one schema — qualify it with its schema`,
+      );
+    }
+    if (!entity) {
+      throw new Error(
+        `Invalid ${label}: subquery must use an entity base view, not '${schema ? schema + '.' : ''}${table}'`,
+      );
+    }
+    if (user) {
+      const perms = entity.GetUserPermisions(user);
+      if (!perms.CanRead) {
+        throw new Error(
+          `Invalid ${label}: you do not have read permission on entity '${entity.Name}' referenced by '${schema ? schema + '.' : ''}${table}'`,
+        );
+      }
+    }
+    return entity;
+  }
+
   /**
    * Optimized RunViewGenericInternal implementation with:
    * - Field filtering at source (Fix #7)
@@ -1068,29 +1205,33 @@ export class ResolverBase {
     aggregates?: AggregateExpression[],
     afterKey?: CompositeKey,
     bypassCache?: boolean,
-    dataSource?: 'Live' | 'Materialized'
+    dataSource?: 'Live' | 'Materialized',
+    includeBinaryFields?: boolean
   ) {
     try {
       if (!viewInfo || !userPayload) return null;
 
+      // Prefer the authenticated session's payload user — it is the authoritative per-request
+      // identity and (for magic-link sessions) carries the per-session resource scope / synthesized
+      // roles that drive RLS. The cached lookup is a fallback for paths where the payload user
+      // isn't populated. For normal users the two are the same instance, so this is a no-op.
+      const user = this.GetUserFromPayload(userPayload)
+        ?? UserCache.Users.find((u) => u.Email.toLowerCase().trim() === userPayload?.email.toLowerCase().trim());
+      if (!user) throw new Error(`User ${userPayload?.email} not found in metadata`);
+
       // SECURITY: GraphQL ExtraFilter may contain IN (SELECT … FROM <base view>).
-      // Screen at this boundary: parse, reject writes, allow only entity BaseViews.
+      // Screen at this boundary: parse, reject writes, allow only entity BaseViews the
+      // acting user can read (subqueries do not inherit the top entity's permission check).
       this.screenClientViewClauses(
         { extraFilter, orderBy, userSearchString, overrideExcludeFilter },
         provider as unknown as IMetadataProvider,
+        user,
       );
 
       // Check API key scope authorization for view operations
       await this.CheckAPIKeyScopeAuthorization('view:run', viewInfo.Entity, userPayload);
 
       const md = provider
-      // Prefer the authenticated session's payload user — it is the authoritative per-request
-      // identity and (for magic-link sessions) carries the per-session resource scope / synthesized
-      // roles that drive RLS. The cached lookup is a fallback for paths where the payload user
-      // isn't populated. For normal users the two are the same instance, so this is a no-op.
-      const user = this.GetUserFromPayload(userPayload)
-        ?? UserCache.Users.find((u) => u.Email.toLowerCase().trim() === userPayload?.email.toLowerCase().trim());
-      if (!user) throw new Error(`User ${userPayload?.email} not found in metadata`);
 
       const entityInfo = md.Entities.find((e) => e.Name === viewInfo.Entity);
       if (!entityInfo) throw new Error(`Entity ${viewInfo.Entity} not found in metadata`);
@@ -1146,6 +1287,7 @@ export class ResolverBase {
           Aggregates: aggregates,
           BypassCache: bypassCache,
           DataSource: dataSource,
+          IncludeBinaryFields: includeBinaryFields,
         },
         user
       );
@@ -1250,6 +1392,7 @@ export class ResolverBase {
             overrideExcludeFilter: param.overrideExcludeFilter,
           },
           md,
+          contextUser ?? undefined,
         );
 
         if (param.viewInfo) {
@@ -1293,6 +1436,7 @@ export class ResolverBase {
           Aggregates: param.aggregates,
           BypassCache: param.bypassCache,
           DataSource: param.dataSource,
+          IncludeBinaryFields: param.includeBinaryFields,
         });
       }
 

package/src/generic/RunViewResolver.ts

@@ -3,7 +3,7 @@ import { AppContext } from '../types.js';
 import { ResolverBase } from './ResolverBase.js';
 import { LogError, LogStatus, EntityInfo, FieldSecurityError, RunViewWithCacheCheckResult, RunViewsWithCacheCheckResponse, RunViewWithCacheCheckParams, AggregateResult, CompositeKey } from '@memberjunction/core';
 
-import { UUIDsEqual } from '@memberjunction/global';
+import { ReplaceByteArraysWithBase64, UUIDsEqual } from '@memberjunction/global';
 import { RequireSystemUser } from '../directives/RequireSystemUser.js';
 import { GetReadOnlyProvider } from '../util.js';
 import { MJUserViewEntityExtended } from '@memberjunction/core-entities';
@@ -200,6 +200,13 @@ export class RunViewByIDInput {
       "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
   })
   DataSource?: 'Live' | 'Materialized';
+
+  @Field(() => Boolean, {
+    nullable: true,
+    description:
+      'Optional, when true the binary fields of the entity (varbinary / bytea columns) are returned, as base64 strings. They are omitted by default because they are usually large and lists rarely read them. A binary field named explicitly in Fields is always returned.',
+  })
+  IncludeBinaryFields?: boolean;
 }
 
 @InputType()
@@ -321,6 +328,13 @@ export class RunViewByNameInput {
       "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
   })
   DataSource?: 'Live' | 'Materialized';
+
+  @Field(() => Boolean, {
+    nullable: true,
+    description:
+      'Optional, when true the binary fields of the entity (varbinary / bytea columns) are returned, as base64 strings. They are omitted by default because they are usually large and lists rarely read them. A binary field named explicitly in Fields is always returned.',
+  })
+  IncludeBinaryFields?: boolean;
 }
 
 @InputType()
@@ -428,6 +442,13 @@ export class RunDynamicViewInput {
       "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
   })
   DataSource?: 'Live' | 'Materialized';
+
+  @Field(() => Boolean, {
+    nullable: true,
+    description:
+      'Optional, when true the binary fields of the entity (varbinary / bytea columns) are returned, as base64 strings. They are omitted by default because they are usually large and lists rarely read them. A binary field named explicitly in Fields is always returned.',
+  })
+  IncludeBinaryFields?: boolean;
 }
 
 @InputType()
@@ -564,6 +585,13 @@ export class RunViewGenericInput {
       "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
   })
   DataSource?: 'Live' | 'Materialized';
+
+  @Field(() => Boolean, {
+    nullable: true,
+    description:
+      'Optional, when true the binary fields of the entity (varbinary / bytea columns) are returned, as base64 strings. They are omitted by default because they are usually large and lists rarely read them. A binary field named explicitly in Fields is always returned.',
+  })
+  IncludeBinaryFields?: boolean;
 }
 
 //****************************************************************************
@@ -1219,6 +1247,7 @@ export class RunViewResolver extends ResolverBase {
           // DataSource must be forwarded on this transport too (same as the InternalRunView/RunViews maps):
           // otherwise a CacheLocal batch's DataSource:'Materialized' request silently reads the live view.
           DataSource: item.params.DataSource,
+          IncludeBinaryFields: item.params.IncludeBinaryFields,
           AfterKey: item.params.AfterKey
             ? CompositeKey.FromKeyValuePairs(item.params.AfterKey.KeyValuePairs)
             : undefined,
@@ -1322,7 +1351,9 @@ export class RunViewResolver extends ResolverBase {
       returnResult.push({
         PrimaryKey: primaryKey,
         EntityID: entityId,
-        Data: JSON.stringify(row),
+        // Binary columns arrive as base64 from the providers; this is the safety net for any
+        // row that still holds a byte array, which JSON.stringify would emit as {type,data}.
+        Data: JSON.stringify(ReplaceByteArraysWithBase64(row)),
       });
     }
     return returnResult;