@memberjunction/server 5.36.0 → 5.38.0

package/src/index.ts

@@ -26,7 +26,9 @@ import sql from 'mssql';
 import { WebSocketServer } from 'ws';
 import buildApolloServer from './apolloServer/index.js';
 import { configInfo, dbDatabase, dbHost, dbPort, dbUsername, graphqlPort, graphqlRootPath, mj_core_schema, websiteRunFromPackage, RESTApiOptions } from './config.js';
+import { default as jwt } from 'jsonwebtoken';
 import { contextFunction, createUnifiedAuthMiddleware, getUserPayload } from './context.js';
+import { UserPayload } from './types.js';
 import { requireSystemUserDirective, publicDirective } from './directives/index.js';
 import createMSSQLConfig from './orm.js';
 import { setupRESTEndpoints } from './rest/setupRESTEndpoints.js';
@@ -667,28 +669,71 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
   const httpServer = createServer(app);
 
   const webSocketServer = new WebSocketServer({ server: httpServer, path: graphqlRootPath });
-  const serverCleanup = useServer(
+
+  // Track per-connection expiry timers so we can clean them up on close
+  const expiryTimers = new WeakMap<object, ReturnType<typeof setTimeout>>();
+
+  const serverCleanup = useServer<Record<string, unknown>, { userPayload: UserPayload }>(
     {
       schema,
-      context: async ({ connectionParams }) => {
-        const userPayload = await getUserPayload(String(connectionParams?.Authorization), undefined, dataSources);
-        return { userPayload };
-      },
-      onError: (ctx, message, errors) => {
-        // Check if error is token expiration (expected behavior)
-        const isTokenExpired = errors.some(err =>
-          err.extensions?.code === 'JWT_EXPIRED' ||
-          err.message?.includes('token has expired')
-        );
+      // RAII: validate the token once at connection time and cache the result.
+      // This prevents re-validating (and re-logging) on every subscription message.
+      onConnect: async (ctx) => {
+        try {
+          const token = String(ctx.connectionParams?.Authorization);
+          const userPayload = await getUserPayload(token, undefined, dataSources);
+
+          // Store validated payload on the connection for use in context()
+          ctx.extra.userPayload = userPayload;
+
+          // Schedule proactive socket close at token expiry so the client
+          // reconnects with a fresh token instead of spamming errors.
+          const decoded = jwt.decode(token.replace('Bearer ', ''));
+          if (decoded && typeof decoded !== 'string' && decoded.exp) {
+            const msUntilExpiry = decoded.exp * 1000 - Date.now();
+            if (msUntilExpiry > 0) {
+              const timer = setTimeout(() => {
+                console.log(`WebSocket token expiring — closing connection for ${userPayload.email}`);
+                // 4403 = Forbidden — retriable in graphql-ws, signals "get a new token"
+                // (4401 is reserved as fatal/"Unauthorized" in the graphql-ws protocol)
+                try {
+                  ctx.extra.socket.close(4403, 'Token expired');
+                } catch {
+                  // Socket may already be closed; benign
+                }
+              }, msUntilExpiry);
+              // Prevent the timer from keeping the process alive during shutdown
+              timer.unref();
+              expiryTimers.set(ctx, timer);
+            }
+          }
 
-        if (isTokenExpired) {
-          // Log at warn level - this is expected from long-lived browser sessions
-          console.warn('WebSocket connection token expired - client should reconnect with refreshed token');
-        } else {
-          // Log actual errors at error level
-          console.error('WebSocket error:', errors);
+          return true;
+        } catch (error: unknown) {
+          // Return false instead of throwing — graphql-ws sends 4403 Forbidden
+          // (retriable) rather than letting the throw produce 1011 (fatal).
+          const msg = error instanceof Error ? error.message : 'Authentication failed';
+          console.warn(`WebSocket connection rejected: ${msg}`);
+          return false;
         }
       },
+      // context() now just reads the already-validated payload — no I/O, no logging
+      context: async (ctx) => {
+        return { userPayload: ctx.extra.userPayload as UserPayload };
+      },
+      onClose: (ctx) => {
+        // Clear the expiry timer if the connection closes before the token expires
+        const timer = expiryTimers.get(ctx);
+        if (timer) {
+          clearTimeout(timer);
+          expiryTimers.delete(ctx);
+        }
+      },
+      onError: (_ctx, _message, errors) => {
+        // Token expiry errors can't happen anymore (handled in onConnect + timer),
+        // so all errors here are genuine and worth logging.
+        console.error('WebSocket error:', errors);
+      },
     },
     webSocketServer
   );

package/src/resolvers/AdhocQueryResolver.ts

@@ -1,8 +1,9 @@
 import { Arg, Ctx, Query, Resolver, Field, Int, InputType } from 'type-graphql';
-import { LogError } from '@memberjunction/core';
+import { DatabasePlatform, LogError } from '@memberjunction/core';
 import { SQLExpressionValidator } from '@memberjunction/global';
+import { RenderPipeline } from '@memberjunction/generic-database-provider';
 import { AppContext } from '../types.js';
-import { GetReadOnlyDataSource } from '../util.js';
+import { GetReadOnlyDataSource, GetReadOnlyProvider } from '../util.js';
 import { ResolverBase } from '../generic/ResolverBase.js';
 import { RunQueryResultType } from './QueryResolver.js';
 import sql from 'mssql';
@@ -19,10 +20,10 @@ class AdhocQueryInput {
     @Field(() => Int, { nullable: true, description: 'Query timeout in seconds. Defaults to 30.' })
     TimeoutSeconds?: number;
 
-    @Field(() => Int, { nullable: true, description: 'Maximum number of rows to return. Applied in-memory after SQL execution; SQL still runs unbounded server-side.' })
+    @Field(() => Int, { nullable: true, description: 'Maximum number of rows to return; applied at the database via the render pipeline.' })
     MaxRows?: number;
 
-    @Field(() => Int, { nullable: true, description: 'Zero-based offset for pagination. Used in conjunction with MaxRows.' })
+    @Field(() => Int, { nullable: true, description: 'Zero-based offset for pagination. When > 0, the row cap switches to OFFSET/FETCH pagination.' })
     StartRow?: number;
 }
 
@@ -62,25 +63,45 @@ export class AdhocQueryResolver extends ResolverBase {
                 return this.buildErrorResult('No read-only data source available for ad-hoc query execution');
             }
 
-            // 3. Build executable SQL. When MaxRows is provided, wrap in a derived table
-            // with outer TOP so the engine can short-circuit at the source instead of
-            // scanning the full result. Skipped for SQL that begins with WITH/CTE — those
-            // can't be nested in a derived table on SQL Server and fall through to the
-            // in-memory slice below.
+            // 3. Resolve platform from the read-only provider for the render pipeline.
+            let platform: DatabasePlatform = 'sqlserver';
+            try {
+                const provider = GetReadOnlyProvider(context.providers, { allowFallbackToReadWrite: false });
+                if (provider?.PlatformKey) platform = provider.PlatformKey;
+            } catch {
+                // Provider not configured — keep the default platform.
+            }
+            const contextUser = context.userPayload?.userRecord;
+
+            // 4. Route the SQL through RenderPipeline so composition tokens
+            // resolve, comments and templates are processed, and the row cap
+            // is applied at the database (via TOP / LIMIT / OFFSET-FETCH).
             const startRow = input.StartRow ?? 0;
             const maxRows = input.MaxRows;
-            const canWrap =
+            const usePaging =
                 maxRows != null &&
                 Number.isInteger(maxRows) &&
                 maxRows > 0 &&
                 Number.isInteger(startRow) &&
-                startRow >= 0 &&
-                !/^\s*WITH\b/i.test(input.SQL);
-            const executableSql = canWrap
-                ? `SELECT TOP ${startRow + maxRows} * FROM (\n${input.SQL}\n) AS _adhoc_capped`
-                : input.SQL;
+                startRow > 0;
+            let executableSql: string;
+            try {
+                const rendered = RenderPipeline.Run(input.SQL, {
+                    Platform: platform,
+                    ContextUser: contextUser,
+                    ...(usePaging
+                        ? { Paging: { StartRow: startRow, MaxRows: maxRows! } }
+                        : maxRows != null && maxRows > 0
+                            ? { MaxRows: maxRows }
+                            : {}),
+                });
+                executableSql = rendered.FinalSQL;
+            } catch (renderErr) {
+                const renderMsg = renderErr instanceof Error ? renderErr.message : String(renderErr);
+                return this.buildErrorResult(`Ad-hoc query rendering failed: ${renderMsg}`);
+            }
 
-            // 4. Execute with timeout
+            // 5. Execute with timeout
             const timeoutMs = (input.TimeoutSeconds ?? 30) * 1000;
             const request = new sql.Request(readOnlyDS);
 
@@ -92,23 +113,16 @@ export class AdhocQueryResolver extends ResolverBase {
             ]);
             const executionTimeMs = Date.now() - startTime;
 
-            // 5. Apply in-memory pagination. With the wrap applied this is a no-op for
-            // first-page reads; for StartRow > 0 (or CTE-headed SQL where the wrap was
-            // skipped) it carves out the requested page.
-            const fullRecordset = result.recordset ?? [];
-            const totalRowCount = fullRecordset.length;
-            let paginated = fullRecordset;
-            if (startRow > 0) paginated = paginated.slice(startRow);
-            if (maxRows != null && maxRows > 0) paginated = paginated.slice(0, maxRows);
-
             // 6. Return as RunQueryResultType
+            const recordset = result.recordset ?? [];
+
             return {
                 QueryID: '',
                 QueryName: 'Ad-Hoc Query',
                 Success: true,
-                Results: JSON.stringify(paginated),
-                RowCount: paginated.length,
-                TotalRowCount: totalRowCount,
+                Results: JSON.stringify(recordset),
+                RowCount: recordset.length,
+                TotalRowCount: recordset.length,
                 PageNumber: maxRows != null && maxRows > 0 ? Math.floor(startRow / maxRows) + 1 : undefined,
                 PageSize: maxRows ?? undefined,
                 ExecutionTime: executionTimeMs,

package/src/resolvers/QueryResolver.ts

@@ -1,9 +1,10 @@
 import { Arg, Ctx, ObjectType, Query, Resolver, Field, Int, InputType } from 'type-graphql';
-import { RunQuery, QueryInfo, IRunQueryProvider, IMetadataProvider, RunQueryParams, LogError, RunQueryWithCacheCheckParams, RunQueriesWithCacheCheckResponse, RunQueryWithCacheCheckResult } from '@memberjunction/core';
+import { RunQuery, IRunQueryProvider, IMetadataProvider, RunQueryParams, LogError, RunQueryWithCacheCheckParams, RunQueriesWithCacheCheckResponse, RunQueryWithCacheCheckResult } from '@memberjunction/core';
 import { AppContext } from '../types.js';
 import { RequireSystemUser } from '../directives/RequireSystemUser.js';
 import { GraphQLJSONObject } from 'graphql-type-json';
 import { Metadata } from '@memberjunction/core';
+import { MJQueryEntity, QueryEngine } from '@memberjunction/core-entities';
 import { GetReadOnlyProvider } from '../util.js';
 import { SQLServerDataProvider } from '@memberjunction/sqlserver-dataprovider';
 import { ResolverBase } from '../generic/ResolverBase.js';
@@ -149,9 +150,9 @@ export class RunQueriesWithCacheCheckOutput {
 
 @Resolver()
 export class RunQueryResolver extends ResolverBase {
-  private async findQuery(md: IMetadataProvider, QueryID: string, QueryName?: string, CategoryID?: string, CategoryPath?: string, refreshMetadataIfNotFound: boolean = false): Promise<QueryInfo | null> {
-    // Filter queries based on provided criteria
-    const queries = md.Queries.filter(q => {
+  private async findQuery(md: IMetadataProvider, QueryID: string, QueryName?: string, CategoryID?: string, CategoryPath?: string, refreshMetadataIfNotFound: boolean = false): Promise<MJQueryEntity | null> {
+    const qe = QueryEngine.Instance;
+    const queries = qe.Queries.filter(q => {
       if (QueryID) {
         return q.ID.trim().toLowerCase() === QueryID.trim().toLowerCase();
       } else if (QueryName) {
@@ -169,8 +170,8 @@ export class RunQueryResolver extends ResolverBase {
 
     if (queries.length === 0) {
       if (refreshMetadataIfNotFound) {
-        // If we didn't find the query, refresh metadata and try again
-        await md.Refresh();
+        // If we didn't find the query, force-refresh QueryEngine and retry
+        await QueryEngine.Instance.Config(true);
         return this.findQuery(md, QueryID, QueryName, CategoryID, CategoryPath, false); // change the refresh flag to false so we don't loop infinitely
       } 
       else {

package/src/resolvers/QuerySystemUserResolver.ts

@@ -1,8 +1,8 @@
 import { Arg, Ctx, Field, InputType, Mutation, ObjectType, registerEnumType, Resolver, PubSub, PubSubEngine } from 'type-graphql';
 import { AppContext } from '../types.js';
-import { LogError, RunView, UserInfo, CompositeKey, DatabaseProviderBase, LogStatus, QueryFieldInfo, QueryParameterInfo, QueryEntityInfo, QueryPermissionInfo } from '@memberjunction/core';
+import { LogError, UserInfo, CompositeKey, DatabaseProviderBase, LogStatus } from '@memberjunction/core';
 import { RequireSystemUser } from '../directives/RequireSystemUser.js';
-import { MJQueryCategoryEntity, MJQueryPermissionEntity } from '@memberjunction/core-entities';
+import { MJQueryCategoryEntity, MJQueryPermissionEntity, MJQueryFieldEntity, MJQueryParameterEntity, MJQueryEntityEntity, QueryEngine } from '@memberjunction/core-entities';
 import { MJQueryResolver, MJQuery_, MJQueryField_, MJQueryParameter_, MJQueryEntity_, MJQueryPermission_ } from '../generated/generated.js';
 import { GetReadWriteProvider } from '../util.js';
 import { DeleteOptionsInput } from '../generic/DeleteOptionsInput.js';
@@ -301,12 +301,9 @@ export class MJQueryResolverExtended extends MJQueryResolver {
 
                 if (input.Permissions && input.Permissions.length > 0) {
                     await this.createPermissions(provider, input.Permissions, queryID, context.userPayload.userRecord);
-                    await record.RefreshRelatedMetadata(true); // force DB update since we just created new permissions
+
                 }
 
-                // Refresh metadata cache to include the newly created query
-                // This ensures subsequent operations can find the query without additional DB calls
-                await provider.Refresh();
 
                 return this.buildSuccessResult(record);
             }
@@ -365,7 +362,7 @@ export class MJQueryResolverExtended extends MJQueryResolver {
         };
     }
 
-    private mapFields(fields: QueryFieldInfo[]): MJQueryField_[] {
+    private mapFields(fields: MJQueryFieldEntity[]): MJQueryField_[] {
         return fields.map(f => ({
             ID: f.ID,
             QueryID: f.QueryID,
@@ -389,7 +386,7 @@ export class MJQueryResolverExtended extends MJQueryResolver {
         }) as MJQueryField_);
     }
 
-    private mapParameters(params: QueryParameterInfo[]): MJQueryParameter_[] {
+    private mapParameters(params: MJQueryParameterEntity[]): MJQueryParameter_[] {
         return params.map(p => ({
             ID: p.ID,
             QueryID: p.QueryID,
@@ -408,7 +405,7 @@ export class MJQueryResolverExtended extends MJQueryResolver {
         }) as MJQueryParameter_);
     }
 
-    private mapEntities(entities: QueryEntityInfo[]): MJQueryEntity_[] {
+    private mapEntities(entities: MJQueryEntityEntity[]): MJQueryEntity_[] {
         return entities.map(e => ({
             ID: e.ID,
             QueryID: e.QueryID,
@@ -422,7 +419,7 @@ export class MJQueryResolverExtended extends MJQueryResolver {
         }) as MJQueryEntity_);
     }
 
-    private mapPermissions(permissions: QueryPermissionInfo[]): MJQueryPermission_[] {
+    private mapPermissions(permissions: MJQueryPermissionEntity[]): MJQueryPermission_[] {
         return permissions.map(p => ({
             ID: p.ID,
             QueryID: p.QueryID,
@@ -528,25 +525,14 @@ export class MJQueryResolverExtended extends MJQueryResolver {
 
             // Handle permissions update if provided
             if (input.Permissions !== undefined) {
-                // Delete existing permissions
-                const rv = new RunView();
-                const existingPermissions = await rv.RunView<MJQueryPermissionEntity>({
-                    EntityName: 'MJ: Query Permissions',
-                    ExtraFilter: `QueryID='${queryID}'`,
-                    ResultType: 'entity_object'
-                }, context.userPayload.userRecord);
-
-                if (existingPermissions.Success && existingPermissions.Results) {
-                    for (const perm of existingPermissions.Results) {
-                        await perm.Delete();
-                    }
+                // Delete existing permissions (read from QueryEngine cache)
+                const existingPermissions = QueryEngine.Instance.GetQueryPermissions(queryID);
+                for (const perm of existingPermissions) {
+                    await perm.Delete();
                 }
 
                 // Create new permissions
                 await this.createPermissions(provider, input.Permissions, queryID, context.userPayload.userRecord);
-                
-                // Refresh the metadata to get updated permissions
-                await queryEntity.RefreshRelatedMetadata(true);
             }
 
             return this.buildSuccessResult(queryEntity);

package/src/resolvers/RunAIAgentResolver.ts

@@ -1298,13 +1298,13 @@ export class RunAIAgentResolver extends ResolverBase {
         // Reverse to get chronological order (oldest first)
         const details = detailsResult.Results.reverse();
 
-        // Get all message IDs for batch loading attachments
+        // Get all message IDs for batch loading artifacts
         const messageIds = details.map(d => d.ID);
 
-        // Batch load all attachments for these messages
-        const attachmentsByDetailId = await attachmentService.GetAttachmentsBatch(messageIds, contextUser, provider);
-
-        // Batch load input artifacts for these messages
+        // Batch load input artifacts for these messages. Since the backfill migration
+        // (V202605271400__Backfill_Attachment_Artifacts) converted all legacy
+        // ConversationDetailAttachment rows to artifact pairs, the artifact junction
+        // is the single source of truth — no separate attachment query needed.
         const inputArtifactsByDetailId = await this.loadInputArtifactsBatch(messageIds, contextUser, provider);
 
         // Build ChatMessage array with attachments and input artifacts
@@ -1312,69 +1312,9 @@ export class RunAIAgentResolver extends ResolverBase {
 
         for (const detail of details) {
             const role = this.mapDetailRoleToMessageRole(detail.Role);
-            const attachments = attachmentsByDetailId.get(detail.ID) || [];
-
-            // Get attachment data with content URLs (handles both inline and FileID storage)
-            const attachmentDataPromises = attachments.map(att =>
-                attachmentService.GetAttachmentData(att, contextUser, provider)
-            );
-            const attachmentDataResults = await Promise.all(attachmentDataPromises);
-
-            // Decide inline vs. tools per-attachment via the artifact-type registry
-            // and the pure RouteArtifact() function. See plans/artifact-attachment-unification.md.
-            //
-            // Note on the modality check: this resolver doesn't know which model will
-            // ultimately run, so RouteArtifact's modality predicate is passed as a no-op.
-            // Model-specific modality enforcement is the driver layer's responsibility;
-            // a follow-up PR can thread the active model through here to surface modality
-            // mismatches at the hard-error path described in plan §4.
             const validAttachments: AttachmentData[] = [];
 
-            for (const result of attachmentDataResults) {
-                if (!result) continue;
-                // Storage-unified attachments link forward to their artifact version
-                // via ArtifactVersionID; the artifact path handles delivery, so skip
-                // the attachment row here to avoid double-processing.
-                if (result.attachment.ArtifactVersionID) {
-                    continue;
-                }
-                const mime = result.attachment.MimeType || '';
-                const fileName = result.attachment.FileName ?? '';
-                const ext = fileName.includes('.') ? fileName.split('.').pop() : undefined;
-                const artifactType = ArtifactMetadataEngine.Instance.GetArtifactTypeByMimeType(mime, ext);
-
-                const decision = RouteArtifact({
-                    typeDefault: artifactType?.DefaultDeliveryMode ?? 'ToolsOnly',
-                    forceToolsOnly: false,
-                    mimeType: mime,
-                    sizeBytes: result.attachment.FileSizeBytes ?? 0,
-                    inlineSizeCap: INLINE_SIZE_CAP,
-                    modelSupportsModality: () => true,
-                    modelName: '<resolver>',
-                    artifactTypeName: artifactType?.Name ?? mime,
-                });
-
-                if (decision.delivery !== 'inline') {
-                    // Tools or error: skip inline embedding. The agent reaches the
-                    // bytes via artifact tools. Driver layer handles modality enforcement.
-                    if (decision.delivery === 'tools' && decision.annotation) {
-                        LogStatus(`[RunAIAgentResolver] ${decision.annotation}`);
-                    }
-                    continue;
-                }
-                validAttachments.push({
-                    type: ConversationUtility.GetAttachmentTypeFromMime(result.attachment.MimeType),
-                    mimeType: result.attachment.MimeType,
-                    fileName: result.attachment.FileName ?? undefined,
-                    sizeBytes: result.attachment.FileSizeBytes ?? undefined,
-                    width: result.attachment.Width ?? undefined,
-                    height: result.attachment.Height ?? undefined,
-                    durationSeconds: result.attachment.DurationSeconds ?? undefined,
-                    content: result.contentUrl
-                });
-            }
-
-            // Get input artifacts for this message — same routing logic, plus ForceToolsOnly.
+            // Get input artifacts for this message — routing via RouteArtifact.
             const inputArtifacts = inputArtifactsByDetailId.get(detail.ID) || [];
             for (const artifactVersion of inputArtifacts) {
                 const artifactMime = artifactVersion.MimeType || '';