@memberjunction/server 2.106.0 → 2.108.0

package/src/agents/skip-sdk.ts

@@ -16,12 +16,15 @@ import {
     SkipAPIRequestAPIKey,
     SkipQueryInfo,
     SkipEntityInfo,
+    SkipEntityFieldInfo,
+    SkipEntityFieldValueInfo,
+    SkipEntityRelationshipInfo,
     SkipAPIAgentNote,
     SkipAPIAgentNoteType,
     SkipAPIArtifact
 } from '@memberjunction/skip-types';
 import { DataContext } from '@memberjunction/data-context';
-import { UserInfo, LogStatus, LogError, Metadata, RunView } from '@memberjunction/core';
+import { UserInfo, LogStatus, LogError, Metadata, RunView, EntityInfo, EntityFieldInfo, EntityRelationshipInfo } from '@memberjunction/core';
 import { sendPostRequest } from '../util.js';
 import { configInfo, baseUrl, publicUrl, graphqlPort, graphqlRootPath, apiKey as callbackAPIKey } from '../config.js';
 import { GetAIAPIKey } from '@memberjunction/ai';
@@ -29,6 +32,8 @@ import { CopyScalarsAndArrays } from '@memberjunction/global';
 import mssql from 'mssql';
 import { registerAccessToken, GetDataAccessToken } from '../resolvers/GetDataResolver.js';
 import { ConversationArtifactEntity, ConversationArtifactVersionEntity, ArtifactTypeEntity } from '@memberjunction/core-entities';
+import { BehaviorSubject } from 'rxjs';
+import { take } from 'rxjs/operators';
 
 /**
  * Configuration options for Skip SDK
@@ -162,6 +167,10 @@ export interface SkipCallResult {
 export class SkipSDK {
     private config: SkipSDKConfig;
 
+    // Static cache for Skip entities (shared across all instances)
+    private static __skipEntitiesCache$: BehaviorSubject<Promise<SkipEntityInfo[]> | null> = new BehaviorSubject<Promise<SkipEntityInfo[]> | null>(null);
+    private static __lastRefreshTime: number = 0;
+
     constructor(config?: SkipSDKConfig) {
         // Use provided config or fall back to MJ server config
         this.config = {
@@ -261,9 +270,12 @@ export class SkipSDK {
         // Build artifacts for this conversation
         const artifacts = await this.buildArtifacts(contextUser, dataSource, conversationId);
 
+        // Process messages: filter delegation messages and enrich with metadata
+        const processedMessages = this.processMessages(messages);
+
         // Construct the full Skip API request
         const request: SkipAPIRequest = {
-            messages,
+            messages: processedMessages,
             conversationID: conversationId,
             dataContext: dataContext ? CopyScalarsAndArrays(dataContext) as DataContext : undefined,
             requestPhase,
@@ -338,13 +350,26 @@ export class SkipSDK {
 
     /**
      * Build entity metadata for Skip
-     * This can be extracted from AskSkipResolver.BuildSkipEntities if needed
+     * Copied from AskSkipResolver.BuildSkipEntities - uses cached metadata with refresh logic
      */
-    private async buildEntities(dataSource: mssql.ConnectionPool, forceRefresh: boolean): Promise<SkipEntityInfo[]> {
-        // TODO: Implement entity metadata building
-        // This would typically query the metadata and convert to SkipEntityInfo format
-        // For now, returning empty array - can be populated later
-        return [];
+    private async buildEntities(dataSource: mssql.ConnectionPool, forceRefresh: boolean, refreshIntervalMinutes: number = 15): Promise<SkipEntityInfo[]> {
+        try {
+            const now = Date.now();
+            const cacheExpired = (now - SkipSDK.__lastRefreshTime) > (refreshIntervalMinutes * 60 * 1000);
+
+            // If force refresh is requested OR cache expired OR cache is empty, refresh
+            if (forceRefresh || cacheExpired || SkipSDK.__skipEntitiesCache$.value === null) {
+                LogStatus(`[SkipSDK] Refreshing Skip entities cache (force: ${forceRefresh}, expired: ${cacheExpired})`);
+                const newData = this.refreshSkipEntities(dataSource);
+                SkipSDK.__skipEntitiesCache$.next(newData);
+            }
+
+            return SkipSDK.__skipEntitiesCache$.pipe(take(1)).toPromise();
+        }
+        catch (e) {
+            LogError(`[SkipSDK] buildEntities error: ${e}`);
+            return [];
+        }
     }
 
     /**
@@ -538,4 +563,322 @@ export class SkipSDK {
             'Content-Type': 'application/json'
         };
     }
+
+    /**
+     * Refreshes the Skip entities cache
+     * Rebuilds the entity information that is provided to Skip
+     * Copied from AskSkipResolver.refreshSkipEntities
+     */
+    private async refreshSkipEntities(dataSource: mssql.ConnectionPool): Promise<SkipEntityInfo[]> {
+        try {
+            const md = new Metadata();
+            const skipSpecialIncludeEntities = (configInfo.askSkip?.entitiesToSend?.includeEntitiesFromExcludedSchemas ?? [])
+                .map((e) => e.trim().toLowerCase());
+
+            // Get the list of entities
+            const entities = md.Entities.filter((e) => {
+                if (!configInfo.askSkip.entitiesToSend.excludeSchemas.includes(e.SchemaName) ||
+                    skipSpecialIncludeEntities.includes(e.Name.trim().toLowerCase())) {
+                    const sd = e.ScopeDefault?.trim();
+                    if (sd && sd.length > 0) {
+                        const scopes = sd.split(',').map((s) => s.trim().toLowerCase()) ?? ['all'];
+                        return !scopes || scopes.length === 0 || scopes.includes('all') || scopes.includes('ai') || skipSpecialIncludeEntities.includes(e.Name.trim().toLowerCase());
+                    }
+                    else {
+                        return true; // no scope, so include it
+                    }
+                }
+                return false;
+            });
+
+            // Now we have our list of entities, pack em up
+            const result = await Promise.all(entities.map((e) => this.packSingleSkipEntityInfo(e, dataSource)));
+
+            SkipSDK.__lastRefreshTime = Date.now(); // Update last refresh time
+            return result;
+        }
+        catch (e) {
+            LogError(`[SkipSDK] refreshSkipEntities error: ${e}`);
+            return [];
+        }
+    }
+
+    /**
+     * Packs information about a single entity for Skip
+     * Includes fields, relationships, and sample data
+     * Copied from AskSkipResolver.PackSingleSkipEntityInfo
+     */
+    private async packSingleSkipEntityInfo(e: EntityInfo, dataSource: mssql.ConnectionPool): Promise<SkipEntityInfo> {
+        try {
+            const ret: SkipEntityInfo = {
+                id: e.ID,
+                name: e.Name,
+                schemaName: e.SchemaName,
+                baseView: e.BaseView,
+                description: e.Description,
+
+                fields: await Promise.all(e.Fields.filter(f => {
+                    // we want to check the scopes for the field level and make sure it is either All or AI or has both
+                    const scopes = f.ScopeDefault?.split(',').map((s) => s.trim().toLowerCase());
+                    return !scopes || scopes.length === 0 || scopes.includes('all') || scopes.includes('ai');
+                }).map(f => {
+                    return this.packSingleSkipEntityField(f, dataSource);
+                })),
+
+                relatedEntities: e.RelatedEntities.map((r) => {
+                    return this.packSingleSkipEntityRelationship(r);
+                }),
+
+                rowsPacked: e.RowsToPackWithSchema,
+                rowsSampleMethod: e.RowsToPackSampleMethod,
+                rows: await this.packEntityRows(e, dataSource)
+            };
+            return ret;
+        }
+        catch (e) {
+            LogError(`[SkipSDK] packSingleSkipEntityInfo error: ${e}`);
+            return null;
+        }
+    }
+
+    /**
+     * Packs information about a single entity relationship
+     * These relationships help Skip understand the data model
+     * Copied from AskSkipResolver.PackSingleSkipEntityRelationship
+     */
+    private packSingleSkipEntityRelationship(r: EntityRelationshipInfo): SkipEntityRelationshipInfo {
+        try {
+            return {
+                entityID: r.EntityID,
+                relatedEntityID: r.RelatedEntityID,
+                type: r.Type,
+                entityKeyField: r.EntityKeyField,
+                relatedEntityJoinField: r.RelatedEntityJoinField,
+                joinView: r.JoinView,
+                joinEntityJoinField: r.JoinEntityJoinField,
+                joinEntityInverseJoinField: r.JoinEntityInverseJoinField,
+                entity: r.Entity,
+                entityBaseView: r.EntityBaseView,
+                relatedEntity: r.RelatedEntity,
+                relatedEntityBaseView: r.RelatedEntityBaseView,
+            };
+        }
+        catch (e) {
+            LogError(`[SkipSDK] packSingleSkipEntityRelationship error: ${e}`);
+            return null;
+        }
+    }
+
+    /**
+     * Packs information about a single entity field
+     * Includes metadata and possible values
+     * Copied from AskSkipResolver.PackSingleSkipEntityField
+     */
+    private async packSingleSkipEntityField(f: EntityFieldInfo, dataSource: mssql.ConnectionPool): Promise<SkipEntityFieldInfo> {
+        try {
+            return {
+                entityID: f.EntityID,
+                sequence: f.Sequence,
+                name: f.Name,
+                displayName: f.DisplayName,
+                category: f.Category,
+                type: f.Type,
+                description: f.Description,
+                isPrimaryKey: f.IsPrimaryKey,
+                allowsNull: f.AllowsNull,
+                isUnique: f.IsUnique,
+                length: f.Length,
+                precision: f.Precision,
+                scale: f.Scale,
+                sqlFullType: f.SQLFullType,
+                defaultValue: f.DefaultValue,
+                autoIncrement: f.AutoIncrement,
+                valueListType: f.ValueListType,
+                extendedType: f.ExtendedType,
+                defaultInView: f.DefaultInView,
+                defaultColumnWidth: f.DefaultColumnWidth,
+                isVirtual: f.IsVirtual,
+                isNameField: f.IsNameField,
+                relatedEntityID: f.RelatedEntityID,
+                relatedEntityFieldName: f.RelatedEntityFieldName,
+                relatedEntity: f.RelatedEntity,
+                relatedEntitySchemaName: f.RelatedEntitySchemaName,
+                relatedEntityBaseView: f.RelatedEntityBaseView,
+                possibleValues: await this.packFieldPossibleValues(f, dataSource),
+            };
+        }
+        catch (e) {
+            LogError(`[SkipSDK] packSingleSkipEntityField error: ${e}`);
+            return null;
+        }
+    }
+
+    /**
+     * Packs entity rows (sample data)
+     * Copied from AskSkipResolver.PackEntityRows
+     */
+    private async packEntityRows(e: EntityInfo, dataSource: mssql.ConnectionPool): Promise<any[]> {
+        try {
+            if (e.RowsToPackWithSchema === 'None')
+                return [];
+
+            // only include columns that have a scopes including either All and/or AI or have Null for ScopeDefault
+            const fields = e.Fields.filter((f) => {
+                const scopes = f.ScopeDefault?.split(',').map((s) => s.trim().toLowerCase());
+                return !scopes || scopes.length === 0 || scopes.includes('all') || scopes.includes('ai');
+            }).map(f => `[${f.Name}]`).join(',');
+
+            // now run the query based on the row packing method
+            let sql: string = '';
+            switch (e.RowsToPackWithSchema) {
+                case 'All':
+                    sql = `SELECT ${fields} FROM ${e.SchemaName}.${e.BaseView}`;
+                    break;
+                case 'Sample':
+                    switch (e.RowsToPackSampleMethod) {
+                        case 'random':
+                            sql = `SELECT TOP ${e.RowsToPackSampleCount} ${fields} FROM [${e.SchemaName}].[${e.BaseView}] ORDER BY newid()`;
+                            break;
+                        case 'top n':
+                            const orderBy = e.RowsToPackSampleOrder ? ` ORDER BY [${e.RowsToPackSampleOrder}]` : '';
+                            sql = `SELECT TOP ${e.RowsToPackSampleCount} ${fields} FROM [${e.SchemaName}].[${e.BaseView}]${orderBy}`;
+                            break;
+                        case 'bottom n':
+                            const firstPrimaryKey = e.FirstPrimaryKey.Name;
+                            const innerOrderBy = e.RowsToPackSampleOrder ? `[${e.RowsToPackSampleOrder}]` : `[${firstPrimaryKey}] DESC`;
+                            sql = `SELECT * FROM (
+                                        SELECT TOP ${e.RowsToPackSampleCount} ${fields}
+                                        FROM [${e.SchemaName}].[${e.BaseView}]
+                                        ORDER BY ${innerOrderBy}
+                                    ) sub
+                                    ORDER BY [${firstPrimaryKey}] ASC;`;
+                            break;
+                    }
+            }
+            const request = new mssql.Request(dataSource);
+            const result = await request.query(sql);
+            if (!result || !result.recordset) {
+                return [];
+            }
+            else {
+                return result.recordset;
+            }
+        }
+        catch (e) {
+            LogError(`[SkipSDK] packEntityRows error: ${e}`);
+            return [];
+        }
+    }
+
+    /**
+     * Packs possible values for an entity field
+     * These values help Skip understand the domain and valid values for fields
+     * Copied from AskSkipResolver.PackFieldPossibleValues
+     */
+    private async packFieldPossibleValues(f: EntityFieldInfo, dataSource: mssql.ConnectionPool): Promise<SkipEntityFieldValueInfo[]> {
+        try {
+            if (f.ValuesToPackWithSchema === 'None') {
+                return []; // don't pack anything
+            }
+            else if (f.ValuesToPackWithSchema === 'All') {
+                // wants ALL of the distinct values
+                return await this.getFieldDistinctValues(f, dataSource);
+            }
+            else if (f.ValuesToPackWithSchema === 'Auto') {
+                // default setting - pack based on the ValueListType
+                if (f.ValueListTypeEnum === 'List') {
+                    // simple list of values in the Entity Field Values table
+                    return f.EntityFieldValues.map((v) => {
+                        return { value: v.Value, displayValue: v.Value };
+                    });
+                }
+                else if (f.ValueListTypeEnum === 'ListOrUserEntry') {
+                    // could be a user provided value, OR the values in the list of possible values.
+                    // get the distinct list of values from the DB and concat that with the f.EntityFieldValues array - deduped and return
+                    const values = await this.getFieldDistinctValues(f, dataSource);
+                    if (!values || values.length === 0) {
+                        // no result, just return the EntityFieldValues
+                        return f.EntityFieldValues.map((v) => {
+                            return { value: v.Value, displayValue: v.Value };
+                        });
+                    }
+                    else {
+                        return [...new Set([...f.EntityFieldValues.map((v) => {
+                            return { value: v.Value, displayValue: v.Value };
+                        }), ...values])];
+                    }
+                }
+            }
+            return []; // if we get here, nothing to pack
+        }
+        catch (e) {
+            LogError(`[SkipSDK] packFieldPossibleValues error: ${e}`);
+            return [];
+        }
+    }
+
+    /**
+     * Gets distinct values for a field from the database
+     * Used to provide Skip with information about the possible values
+     * Copied from AskSkipResolver.GetFieldDistinctValues
+     */
+    private async getFieldDistinctValues(f: EntityFieldInfo, dataSource: mssql.ConnectionPool): Promise<SkipEntityFieldValueInfo[]> {
+        try {
+            const sql = `SELECT DISTINCT ${f.Name} FROM ${f.SchemaName}.${f.BaseView}`;
+            const request = new mssql.Request(dataSource);
+            const result = await request.query(sql);
+            if (!result || !result.recordset) {
+                return [];
+            }
+            else {
+                return result.recordset.map((r) => {
+                    return {
+                        value: r[f.Name],
+                        displayValue: r[f.Name]
+                    };
+                });
+            }
+        }
+        catch (e) {
+            LogError(`[SkipSDK] getFieldDistinctValues error: ${e}`);
+            return [];
+        }
+    }
+
+    /**
+     * Process messages: filter delegation messages and add metadata fields
+     * Messages coming in should already have conversationDetailID if they exist in the database
+     */
+    private processMessages(messages: SkipMessage[]): SkipMessage[] {
+        // Filter out delegation messages (administrative messages that shouldn't go to Skip)
+        const filteredMessages = messages.filter(msg => !this.isDelegationMessage(msg.content));
+
+        // Enrich messages with default metadata if not already present
+        return filteredMessages.map(msg => ({
+            ...msg,
+            // Add default metadata fields if not already present
+            // Messages from DB already have conversationDetailID, temp messages get temp-X
+            hiddenToUser: msg.hiddenToUser ?? false,
+            userRating: msg.userRating ?? null,
+            userFeedback: msg.userFeedback ?? null,
+            reflectionInsights: msg.reflectionInsights ?? null,
+            summaryOfEarlierConveration: msg.summaryOfEarlierConveration ?? null
+        }));
+    }
+
+    /**
+     * Check if a message is a delegation message that should be filtered out
+     * Uses flexible pattern matching to detect variations of delegation messages
+     */
+    private isDelegationMessage(content: string): boolean {
+        if (!content) return false;
+
+        const lowerContent = content.toLowerCase();
+
+        // Check for both "delegating" or "delegate" AND "skip" in any order
+        const hasDelegatingOrDelegate = lowerContent.includes('delegating') || lowerContent.includes('delegate');
+        const hasSkip = lowerContent.includes('skip');
+
+        return hasDelegatingOrDelegate && hasSkip;
+    }
 }

package/src/config.ts

@@ -130,6 +130,14 @@ const componentRegistrySchema = z.object({
   headers: z.record(z.string()).optional(),
 }).passthrough(); // Allow additional fields
 
+const scheduledJobsSchema = z.object({
+  enabled: z.boolean().optional().default(false),
+  systemUserEmail: z.string().optional().default('system@memberjunction.org'),
+  maxConcurrentJobs: z.number().optional().default(5),
+  defaultLockTimeout: z.number().optional().default(600000), // 10 minutes in ms
+  staleLockCleanupInterval: z.number().optional().default(300000), // 5 minutes in ms
+});
+
 const configInfoSchema = z.object({
   userHandling: userHandlingInfoSchema,
   databaseSettings: databaseSettingsInfoSchema,
@@ -139,6 +147,7 @@ const configInfoSchema = z.object({
   sqlLogging: sqlLoggingSchema.optional(),
   authProviders: z.array(authProviderSchema).optional(),
   componentRegistries: z.array(componentRegistrySchema).optional(),
+  scheduledJobs: scheduledJobsSchema.optional().default({}),
 
   apiKey: z.string().optional(),
   baseUrl: z.string().default('http://localhost'),
@@ -180,6 +189,7 @@ export type SqlLoggingOptions = z.infer<typeof sqlLoggingOptionsSchema>;
 export type SqlLoggingInfo = z.infer<typeof sqlLoggingSchema>;
 export type AuthProviderConfig = z.infer<typeof authProviderSchema>;
 export type ComponentRegistryConfig = z.infer<typeof componentRegistrySchema>;
+export type ScheduledJobsConfig = z.infer<typeof scheduledJobsSchema>;
 export type ConfigInfo = z.infer<typeof configInfoSchema>;
 
 export const configInfo: ConfigInfo = loadConfig();

package/src/context.ts

@@ -145,9 +145,10 @@ export const contextFunction =
     const apiKey = String(req.headers['x-mj-api-key']);
     
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const operationName: string | undefined = (req as any).body?.operationName;
+    const reqAny = req as any;
+    const operationName: string | undefined = reqAny.body?.operationName;
     if (operationName !== 'IntrospectionQuery') {
-      console.log({ operationName });
+      console.log({ operationName, variables: reqAny.body?.variables || undefined });
     }
 
     const userPayload = await getUserPayload(