@memberjunction/server 0.9.164 → 0.9.166

package/src/resolvers/AskSkipResolver.ts

@@ -10,7 +10,7 @@ import { promisify } from 'util';
 const gzip = promisify(zlib.gzip);
 
 import { PUSH_STATUS_UPDATES_TOPIC } from '../generic/PushStatusResolver';
-import { ConversationDetailEntity, ConversationEntity, UserNotificationEntity, UserViewEntityExtended } from '@memberjunction/core-entities';
+import { ConversationDetailEntity, ConversationEntity, DataContextEntity, DataContextItemEntity, UserNotificationEntity, UserViewEntity, UserViewEntityExtended } from '@memberjunction/core-entities';
 import { DataSource } from 'typeorm';
 import { ___skipAPIOrgId, ___skipAPIurl } from '../config';
 
@@ -64,24 +64,83 @@ export class AskSkipResolver {
   @Query(() => AskSkipResultType)
   async ExecuteAskSkipAnalysisQuery(
     @Arg('UserQuestion', () => String) UserQuestion: string,
-    @Arg('ViewId', () => Int) ViewId: number,
     @Arg('ConversationId', () => Int) ConversationId: number,
     @Ctx() { dataSource, userPayload }: AppContext,
-    @PubSub() pubSub: PubSubEngine
+    @PubSub() pubSub: PubSubEngine,
+    @Arg('DataContextId', () => Int, { nullable: true }) DataContextId?: number
   ) {
     const md = new Metadata();
     const user = UserCache.Instance.Users.find((u) => u.Email === userPayload.email);
     if (!user) throw new Error(`User ${userPayload.email} not found in UserCache`);
 
+    const {convoEntity, dataContextEntity, convoDetailEntity, dataContext} = await this.HandleSkipInitialObjectLoading(dataSource, ConversationId, UserQuestion, user, userPayload, md, DataContextId);
+
+    const OrganizationId = ___skipAPIOrgId;
+
+    // now load up the messages. We will load up ALL of the messages for this conversation, and then pass them to the Skip API
+    const messages: SkipMessage[] = await this.LoadConversationDetailsIntoSkipMessages(dataSource, convoEntity.ID, AskSkipResolver._maxHistoricalMessages);
+
+    const input: SkipAPIRequest = { 
+                    messages: messages, 
+                    conversationID: ConversationId.toString(), 
+                    dataContext: dataContext, 
+                    organizationID: !isNaN(parseInt(OrganizationId)) ? parseInt(OrganizationId) : 0,
+                    requestPhase: 'initial_request'
+                  };
+
+    pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
+      message: JSON.stringify({
+        type: 'AskSkip',
+        status: 'OK',
+        message: 'I will be happy to help and will start by analyzing your request...',
+      }),
+      sessionId: userPayload.sessionId,
+    });
+
+    return this.HandleSkipRequest(input, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
+  }
+
+
+  protected async HandleSkipInitialObjectLoading(dataSource: DataSource, 
+                                                 ConversationId: number, 
+                                                 UserQuestion: string, 
+                                                 user: UserInfo,
+                                                 userPayload: UserPayload, 
+                                                 md: Metadata,
+                                                 DataContextId: number): Promise<{convoEntity: ConversationEntity, 
+                                                                         dataContextEntity: DataContextEntity, 
+                                                                         convoDetailEntity: ConversationDetailEntity, 
+                                                                         dataContext: SkipDataContext}> {
     const convoEntity = <ConversationEntity>await md.GetEntityObject('Conversations', user);
+    let dataContextEntity: DataContextEntity;
+
     if (!ConversationId || ConversationId <= 0) {
       // create a new conversation id
       convoEntity.NewRecord();
       if (user) {
         convoEntity.UserID = user.ID;
         convoEntity.Name = AskSkipResolver._defaultNewChatName;
-        if (await convoEntity.Save()) 
+
+        dataContextEntity = await md.GetEntityObject<DataContextEntity>('Data Contexts', user);
+        if (!DataContextId || DataContextId <= 0) {
+          dataContextEntity.NewRecord();
+          dataContextEntity.UserID = user.ID;
+          dataContextEntity.Name = 'Data Context for Skip Conversation';
+          if (!await dataContextEntity.Save())
+            throw new Error(`Creating a new data context failed`);
+        }
+        else {
+          await dataContextEntity.Load(DataContextId);
+        }
+        convoEntity.DataContextID = dataContextEntity.ID;
+        if (await convoEntity.Save()) {
           ConversationId = convoEntity.ID;
+          if (!DataContextId || dataContextEntity.ID <= 0) {
+            // only do this if we created a new data context for this conversation
+            dataContextEntity.Name += ` ${ConversationId}`;
+            await dataContextEntity.Save();  
+          }
+        }
         else 
           throw new Error(`Creating a new conversation failed`);
       } 
@@ -90,8 +149,16 @@ export class AskSkipResolver {
       }
     } else {
       await convoEntity.Load(ConversationId); // load the existing conversation, will need it later
+      dataContextEntity = await md.GetEntityObject<DataContextEntity>('Data Contexts', user);
+
+      // note - we ignore the parameter DataContextId if it is passed in, we will use the data context from the conversation that is saved. If a user wants to change the data context for a convo, they can do that elsewhere
+      if (DataContextId && DataContextId > 0 && DataContextId !== convoEntity.DataContextID) 
+        console.log(`AskSkipResolver: DataContextId ${DataContextId} was passed in but it was ignored because it was different than the DataContextID in the conversation ${convoEntity.DataContextID}`);
+
+      await dataContextEntity.Load(convoEntity.DataContextID);
     }
 
+
     // now, create a conversation detail record for the user message
     const convoDetailEntity = await md.GetEntityObject<ConversationDetailEntity>('Conversation Details', user);
     convoDetailEntity.NewRecord();
@@ -101,38 +168,65 @@ export class AskSkipResolver {
     convoDetailEntity.Set('Sequence', 1); // using weakly typed here because we're going to get rid of this field soon
     await convoDetailEntity.Save();
 
-    //const OrganizationId = 2 //HG 8/1/2023 TODO: Pull this from an environment variable
-    const OrganizationId = ___skipAPIOrgId;
     const dataContext: SkipDataContext = new SkipDataContext();
-    dataContext.Items.push(
-      {
-        Type: 'view',
-        RecordID: ViewId,
-        Data: await this.getViewData(ViewId, user),
-      } as SkipDataContextItem
-    );    
+    const dciEntityInfo = md.Entities.find((e) => e.Name === 'Data Context Items');
+    if (!dciEntityInfo)
+      throw new Error(`Data Context Items entity not found`);
+
+    const sql = `SELECT * FROM ${dciEntityInfo.SchemaName}.${dciEntityInfo.BaseView} WHERE DataContextID = ${dataContextEntity.ID}`;
+    const result = await dataSource.query(sql);
+    if (!result) 
+      throw new Error(`Error running SQL: ${sql}`);
+    else {
+      for (const r of result) {
+        const item = new SkipDataContextItem();
+        item.Type = r.Type;
+        switch (item.Type) {
+          case 'full_entity':
+            item.EntityID = r.EntityID;  
+            break;
+          case 'single_record':
+            item.RecordID = r.RecordID;  
+            item.EntityID = r.EntityID;  
+            break;
+          case 'query':
+            item.QueryID = r.QueryID; // map the QueryID in our database to the RecordID field in the object model for runtime use
+            break;
+          case 'sql':
+            item.SQL = r.SQL;  
+            break;
+          case 'view':
+            item.ViewID = r.ViewID;
+            item.EntityID = r.EntityID;
+            break;
+        }
+        if (item.EntityID) {
+          item.Entity = md.Entities.find((e) => e.ID === item.EntityID);
+          item.EntityName = item.Entity.Name;
+          if (item.Type === 'full_entity')
+            item.RecordName = item.EntityName;
+        }
+        if (item.Type === 'query' && item.QueryID) {
+          const q = md.Queries.find((q) => q.ID === item.QueryID);
+          item.RecordName = q?.Name;
+        }
+        if (item.Type === 'view' && item.ViewID) {
+          const v = await md.GetEntityObject<UserViewEntityExtended>('User Views', user);
+          await v.Load(item.ViewID);
+          item.RecordName = v.Name;
+          item.ViewEntity = v;
+        }
+        item.Data = r.Data && r.Data.length > 0 ? JSON.parse(r.Data) : item.Data; // parse the stored data if it was saved, otherwise leave it to whatever the object's default is
+        item.AdditionalDescription = r.AdditionalDescription;
+        item.DataContextItemID = r.ID;
+        dataContext.Items.push(item);
+      }
+    }
 
-    // now load up the messages. We will load up ALL of the messages for this conversation, and then pass them to the Skip API
-    const messages: SkipMessage[] = await this.LoadConversationDetailsIntoSkipMessages(dataSource, convoEntity.ID, AskSkipResolver._maxHistoricalMessages);
 
-    const input: SkipAPIRequest = { 
-                    messages: messages, 
-                    conversationID: ConversationId.toString(), 
-                    dataContext: dataContext, 
-                    organizationID: !isNaN(parseInt(OrganizationId)) ? parseInt(OrganizationId) : 0,
-                    requestPhase: 'initial_request'
-                  };
-
-    pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
-      message: JSON.stringify({
-        type: 'AskSkip',
-        status: 'OK',
-        message: 'I will be happy to help and will start by analyzing your request...',
-      }),
-      sessionId: userPayload.sessionId,
-    });
+    /// TODO   next up we need to modify MJExplorer to handle the data context stuff and then we can finish this method
 
-    return this.HandleSkipRequest(input, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity);
+    return {dataContext, convoEntity, dataContextEntity, convoDetailEntity};
   }
 
   protected async LoadConversationDetailsIntoSkipMessages(dataSource: DataSource, ConversationId: number, maxHistoricalMessages?: number): Promise<SkipMessage[]> {
@@ -191,7 +285,8 @@ export class AskSkipResolver {
 
   protected async HandleSkipRequest(input: SkipAPIRequest, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
                                     ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, md: Metadata, 
-                                    convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                    convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                    dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     LogStatus(`Sending request to Skip API: ${___skipAPIurl}`)
 
     // Convert JSON payload to a Buffer and compress it
@@ -212,14 +307,14 @@ export class AskSkipResolver {
 
       // now, based on the result type, we will either wait for the next phase or we will process the results
       if (apiResponse.responsePhase === 'data_request') {
-        return await this.HandleDataRequestPhase(input, <SkipAPIDataRequestResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
+        return await this.HandleDataRequestPhase(input, <SkipAPIDataRequestResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
       }
       else if (apiResponse.responsePhase === 'clarifying_question') {
         // need to send the request back to the user for a clarifying question
         return await this.HandleClarifyingQuestionPhase(input, <SkipAPIClarifyingQuestionResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
       }
       else if (apiResponse.responsePhase === 'analysis_complete') {
-        return await this.HandleAnalysisComplete(input, <SkipAPIAnalysisCompleteResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
+        return await this.HandleAnalysisComplete(input, <SkipAPIAnalysisCompleteResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
       }
       else {
         // unknown response phase
@@ -274,11 +369,12 @@ export class AskSkipResolver {
   }
 
   protected async HandleAnalysisComplete(apiRequest: SkipAPIRequest, apiResponse: SkipAPIAnalysisCompleteResponse, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
-                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                         dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     // analysis is complete
     // all done, wrap things up
     const md = new Metadata();
-    const {AIMessageConversationDetailID} = await this.FinishConversationAndNotifyUser(apiResponse, md, user, convoEntity, pubSub, userPayload);
+    const {AIMessageConversationDetailID} = await this.FinishConversationAndNotifyUser(apiResponse, dataContext, dataContextEntity, md, user, convoEntity, pubSub, userPayload);
 
     return {
       Success: true,
@@ -325,7 +421,8 @@ export class AskSkipResolver {
   }
 
   protected async HandleDataRequestPhase(apiRequest: SkipAPIRequest, apiResponse: SkipAPIDataRequestResponse, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
-                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                         dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     // our job in this method is to go through each of the data requests from the Skip API, get the data, and then go back to the Skip API again and to the next phase
     try {  
       const _maxDataGatheringRetries = 5;
@@ -352,9 +449,9 @@ export class AskSkipResolver {
               const item = new SkipDataContextItem();
               item.Type = 'sql';
               item.Data = result;
-              item.RecordName = dr.text;
+              item.SQL = dr.text;
               item.AdditionalDescription = dr.description;
-              apiRequest.dataContext.Items.push(item);
+              dataContext.Items.push(item);
               break;
             case "stored_query":
               const queryName = dr.text;
@@ -366,10 +463,10 @@ export class AskSkipResolver {
                   const item = new SkipDataContextItem();
                   item.Type = 'query';
                   item.Data = result.Results;
-                  item.RecordID = query.ID;
+                  item.QueryID = query.ID;
                   item.RecordName = query.Name;
                   item.AdditionalDescription = dr.description;
-                  apiRequest.dataContext.Items.push(item);    
+                  dataContext.Items.push(item);    
                 }
                 else
                   throw new Error(`Error running query ${queryName}`);
@@ -415,7 +512,7 @@ export class AskSkipResolver {
           apiRequest.requestPhase = 'data_gathering_response';
         }
       // we have all of the data now, add it to the data context and then submit it back to the Skip API
-      return this.HandleSkipRequest(apiRequest, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity);
+      return this.HandleSkipRequest(apiRequest, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
     }
     catch (e) {
       LogError(e);
@@ -424,7 +521,7 @@ export class AskSkipResolver {
   }
 
   /**
-   * This method will handle the process for an end of request where a user is notified of an AI message. The AI message is either the finished report or a clarifying question.
+   * This method will handle the process for an end of successful request where a user is notified of an AI message. The AI message is either the finished report or a clarifying question.
    * @param apiResponse 
    * @param md 
    * @param user 
@@ -433,11 +530,11 @@ export class AskSkipResolver {
    * @param userPayload 
    * @returns 
    */
-  protected async FinishConversationAndNotifyUser(apiResponse: SkipAPIAnalysisCompleteResponse, md: Metadata, user: UserInfo, convoEntity: ConversationEntity, pubSub: PubSubEngine, userPayload: UserPayload): Promise<{AIMessageConversationDetailID: number}> {
+  protected async FinishConversationAndNotifyUser(apiResponse: SkipAPIAnalysisCompleteResponse, dataContext: SkipDataContext, dataContextEntity: DataContextEntity, md: Metadata, user: UserInfo, convoEntity: ConversationEntity, pubSub: PubSubEngine, userPayload: UserPayload): Promise<{AIMessageConversationDetailID: number}> {
     const sTitle = apiResponse.reportTitle; 
     const sResult = JSON.stringify(apiResponse);
 
-    // now, create a conversation detail record for the Skip response
+    // Create a conversation detail record for the Skip response
     const convoDetailEntityAI = <ConversationDetailEntity>await md.GetEntityObject('Conversation Details', user);
     convoDetailEntityAI.NewRecord();
     convoDetailEntityAI.ConversationID = convoEntity.ID;
@@ -447,7 +544,7 @@ export class AskSkipResolver {
     await convoDetailEntityAI.Save();
 
     // finally update the convo name if it is still the default
-    if (convoEntity.Name === AskSkipResolver._defaultNewChatName && sTitle) {
+    if (convoEntity.Name === AskSkipResolver._defaultNewChatName && sTitle && sTitle !== AskSkipResolver._defaultNewChatName) {
       convoEntity.Name = sTitle; // use the title from the response
       await convoEntity.Save();
     }
@@ -464,6 +561,39 @@ export class AskSkipResolver {
       conversationId: convoEntity.ID,
     });
     await userNotification.Save();
+    
+    // now, persist the data context items, first let's get 
+    for (const item of dataContext.Items) {
+      const dciEntity = <DataContextItemEntity>await md.GetEntityObject('Data Context Items', user);
+      if (item.DataContextItemID > 0) 
+        await dciEntity.Load(item.DataContextItemID);
+      else
+        dciEntity.NewRecord();
+      dciEntity.DataContextID = dataContextEntity.ID;
+      dciEntity.Type = item.Type;
+      switch (item.Type) {
+        case 'full_entity':
+        case 'single_record':
+          const e = item.Entity || md.Entities.find((e) => e.Name === item.EntityName);
+          dciEntity.EntityID = e.ID;
+          if (item.Type === 'single_record')
+            dciEntity.RecordID = item.RecordID;
+          break;
+        case 'view':
+          dciEntity.ViewID = item.ViewID;  
+          break;
+        case 'query':
+          dciEntity.QueryID = item.QueryID;  
+          break;
+        case 'sql':
+          dciEntity.SQL = item.SQL;  
+          break;
+      }
+      dciEntity.DataJSON = JSON.stringify(item.Data);
+      await dciEntity.Save();
+    }
+
+    // send a UI update trhough pub-sub
     pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
       message: JSON.stringify({
         type: 'UserNotifications',
@@ -475,6 +605,7 @@ export class AskSkipResolver {
       }),
       sessionId: userPayload.sessionId,
     });
+
     return {
       AIMessageConversationDetailID: convoDetailEntityAI.ID
     };