@memberjunction/server 0.9.163 → 0.9.165

package/src/resolvers/AskSkipResolver.ts

@@ -10,7 +10,7 @@ import { promisify } from 'util';
 const gzip = promisify(zlib.gzip);
 
 import { PUSH_STATUS_UPDATES_TOPIC } from '../generic/PushStatusResolver';
-import { ConversationDetailEntity, ConversationEntity, UserNotificationEntity, UserViewEntityExtended } from '@memberjunction/core-entities';
+import { ConversationDetailEntity, ConversationEntity, DataContextEntity, DataContextItemEntity, UserNotificationEntity, UserViewEntityExtended } from '@memberjunction/core-entities';
 import { DataSource } from 'typeorm';
 import { ___skipAPIOrgId, ___skipAPIurl } from '../config';
 
@@ -73,25 +73,80 @@ export class AskSkipResolver {
     const user = UserCache.Instance.Users.find((u) => u.Email === userPayload.email);
     if (!user) throw new Error(`User ${userPayload.email} not found in UserCache`);
 
+    const {convoEntity, dataContextEntity, convoDetailEntity, dataContext} = await this.HandleCreationOfEntityObjects(dataSource, ConversationId, UserQuestion, user, userPayload, md, ViewId);
+
+    const OrganizationId = ___skipAPIOrgId;
+
+    // now load up the messages. We will load up ALL of the messages for this conversation, and then pass them to the Skip API
+    const messages: SkipMessage[] = await this.LoadConversationDetailsIntoSkipMessages(dataSource, convoEntity.ID, AskSkipResolver._maxHistoricalMessages);
+
+    const input: SkipAPIRequest = { 
+                    messages: messages, 
+                    conversationID: ConversationId.toString(), 
+                    dataContext: dataContext, 
+                    organizationID: !isNaN(parseInt(OrganizationId)) ? parseInt(OrganizationId) : 0,
+                    requestPhase: 'initial_request'
+                  };
+
+    pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
+      message: JSON.stringify({
+        type: 'AskSkip',
+        status: 'OK',
+        message: 'I will be happy to help and will start by analyzing your request...',
+      }),
+      sessionId: userPayload.sessionId,
+    });
+
+    return this.HandleSkipRequest(input, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
+  }
+
+  protected async HandleCreationOfEntityObjects(dataSource: DataSource, 
+                                                ConversationId: number, 
+                                                UserQuestion: string, 
+                                                user: UserInfo,
+                                                userPayload: UserPayload, 
+                                                md: Metadata,
+                                                ViewId: number): Promise<{convoEntity: ConversationEntity, 
+                                                                        dataContextEntity: DataContextEntity, 
+                                                                        convoDetailEntity: ConversationDetailEntity, 
+                                                                        dataContext: SkipDataContext}> {
     const convoEntity = <ConversationEntity>await md.GetEntityObject('Conversations', user);
+    let dataContextEntity: DataContextEntity;
+
     if (!ConversationId || ConversationId <= 0) {
       // create a new conversation id
       convoEntity.NewRecord();
       if (user) {
         convoEntity.UserID = user.ID;
         convoEntity.Name = AskSkipResolver._defaultNewChatName;
-        if (await convoEntity.Save()) 
-          ConversationId = convoEntity.ID;
-        else 
-          throw new Error(`Creating a new conversation failed`);
+
+        dataContextEntity = await md.GetEntityObject<DataContextEntity>('Data Contexts', user);
+        dataContextEntity.NewRecord();
+        dataContextEntity.UserID = user.ID;
+        dataContextEntity.Name = 'Data Context for Skip Conversation';
+        if (await dataContextEntity.Save()) {
+          convoEntity.DataContextID = dataContextEntity.ID;
+          if (await convoEntity.Save()) {
+            ConversationId = convoEntity.ID;
+            dataContextEntity.Name += ` ${ConversationId}`;
+            await dataContextEntity.Save();
+          }
+          else 
+            throw new Error(`Creating a new conversation failed`);
+        }
+        else
+          throw new Error(`Creating a new data context failed`);
       } 
       else { 
         throw new Error(`User ${userPayload.email} not found in UserCache`);
       }
     } else {
       await convoEntity.Load(ConversationId); // load the existing conversation, will need it later
+      dataContextEntity = await md.GetEntityObject<DataContextEntity>('Data Contexts', user);
+      await dataContextEntity.Load(convoEntity.DataContextID);
     }
 
+
     // now, create a conversation detail record for the user message
     const convoDetailEntity = await md.GetEntityObject<ConversationDetailEntity>('Conversation Details', user);
     convoDetailEntity.NewRecord();
@@ -101,38 +156,38 @@ export class AskSkipResolver {
     convoDetailEntity.Set('Sequence', 1); // using weakly typed here because we're going to get rid of this field soon
     await convoDetailEntity.Save();
 
-    //const OrganizationId = 2 //HG 8/1/2023 TODO: Pull this from an environment variable
-    const OrganizationId = ___skipAPIOrgId;
     const dataContext: SkipDataContext = new SkipDataContext();
-    dataContext.Items.push(
-      {
-        Type: 'view',
-        RecordID: ViewId,
-        Data: await this.getViewData(ViewId, user),
-      } as SkipDataContextItem
-    );    
-
-    // now load up the messages. We will load up ALL of the messages for this conversation, and then pass them to the Skip API
-    const messages: SkipMessage[] = await this.LoadConversationDetailsIntoSkipMessages(dataSource, convoEntity.ID, AskSkipResolver._maxHistoricalMessages);
-
-    const input: SkipAPIRequest = { 
-                    messages: messages, 
-                    conversationID: ConversationId.toString(), 
-                    dataContext: dataContext, 
-                    organizationID: !isNaN(parseInt(OrganizationId)) ? parseInt(OrganizationId) : 0,
-                    requestPhase: 'initial_request'
-                  };
-
-    pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
-      message: JSON.stringify({
-        type: 'AskSkip',
-        status: 'OK',
-        message: 'I will be happy to help and will start by analyzing your request...',
-      }),
-      sessionId: userPayload.sessionId,
-    });
+    const dciEntityInfo = md.Entities.find((e) => e.Name === 'Data Context Items');
+    if (!dciEntityInfo)
+      throw new Error(`Data Context Items entity not found`);
+
+    const sql = `SELECT * FROM ${dciEntityInfo.SchemaName}.${dciEntityInfo.BaseView} WHERE DataContextID = ${dataContextEntity.ID}`;
+    const result = await dataSource.query(sql);
+    if (!result) 
+      throw new Error(`Error running SQL: ${sql}`);
+    else {
+      for (const r of result) {
+        const item = new SkipDataContextItem();
+        item.Type = r.Type;
+        item.RecordID = r.RecordID;
+        item.RecordName = r.RecordName;
+        item.Data = r.Data && r.Data.length > 0 ? JSON.parse(r.Data) : item.Data; // parse the stored data if it was saved, otherwise leave it to whatever the object's default is
+        item.AdditionalDescription = r.AdditionalDescription;
+        dataContext.Items.push(item);
+      }
+    }
 
-    return this.HandleSkipRequest(input, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity);
+    // now if we don't already have this view in our data context, we will add it
+    if (ViewId && !dataContext.Items.find(i => i.Type === 'view' && i.RecordID === ViewId))
+      dataContext.Items.push(
+        {
+          Type: 'view',
+          RecordID: ViewId,
+          Data: await this.getViewData(ViewId, user),
+        } as SkipDataContextItem
+      );    
+
+    return {dataContext, convoEntity, dataContextEntity, convoDetailEntity};
   }
 
   protected async LoadConversationDetailsIntoSkipMessages(dataSource: DataSource, ConversationId: number, maxHistoricalMessages?: number): Promise<SkipMessage[]> {
@@ -191,7 +246,8 @@ export class AskSkipResolver {
 
   protected async HandleSkipRequest(input: SkipAPIRequest, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
                                     ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, md: Metadata, 
-                                    convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                    convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                    dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     LogStatus(`Sending request to Skip API: ${___skipAPIurl}`)
 
     // Convert JSON payload to a Buffer and compress it
@@ -212,14 +268,14 @@ export class AskSkipResolver {
 
       // now, based on the result type, we will either wait for the next phase or we will process the results
       if (apiResponse.responsePhase === 'data_request') {
-        return await this.HandleDataRequestPhase(input, <SkipAPIDataRequestResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
+        return await this.HandleDataRequestPhase(input, <SkipAPIDataRequestResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
       }
       else if (apiResponse.responsePhase === 'clarifying_question') {
         // need to send the request back to the user for a clarifying question
         return await this.HandleClarifyingQuestionPhase(input, <SkipAPIClarifyingQuestionResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
       }
       else if (apiResponse.responsePhase === 'analysis_complete') {
-        return await this.HandleAnalysisComplete(input, <SkipAPIAnalysisCompleteResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity);
+        return await this.HandleAnalysisComplete(input, <SkipAPIAnalysisCompleteResponse>apiResponse, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
       }
       else {
         // unknown response phase
@@ -274,11 +330,12 @@ export class AskSkipResolver {
   }
 
   protected async HandleAnalysisComplete(apiRequest: SkipAPIRequest, apiResponse: SkipAPIAnalysisCompleteResponse, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
-                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                         dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     // analysis is complete
     // all done, wrap things up
     const md = new Metadata();
-    const {AIMessageConversationDetailID} = await this.FinishConversationAndNotifyUser(apiResponse, md, user, convoEntity, pubSub, userPayload);
+    const {AIMessageConversationDetailID} = await this.FinishConversationAndNotifyUser(apiResponse, dataContext, dataContextEntity, md, user, convoEntity, pubSub, userPayload);
 
     return {
       Success: true,
@@ -325,7 +382,8 @@ export class AskSkipResolver {
   }
 
   protected async HandleDataRequestPhase(apiRequest: SkipAPIRequest, apiResponse: SkipAPIDataRequestResponse, UserQuestion: string, user: UserInfo, dataSource: DataSource, 
-                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity): Promise<AskSkipResultType> {
+                                         ConversationId: number, userPayload: UserPayload, pubSub: PubSubEngine, convoEntity: ConversationEntity, convoDetailEntity: ConversationDetailEntity,
+                                         dataContext: SkipDataContext, dataContextEntity: DataContextEntity): Promise<AskSkipResultType> {
     // our job in this method is to go through each of the data requests from the Skip API, get the data, and then go back to the Skip API again and to the next phase
     try {  
       const _maxDataGatheringRetries = 5;
@@ -354,7 +412,7 @@ export class AskSkipResolver {
               item.Data = result;
               item.RecordName = dr.text;
               item.AdditionalDescription = dr.description;
-              apiRequest.dataContext.Items.push(item);
+              dataContext.Items.push(item);
               break;
             case "stored_query":
               const queryName = dr.text;
@@ -369,7 +427,7 @@ export class AskSkipResolver {
                   item.RecordID = query.ID;
                   item.RecordName = query.Name;
                   item.AdditionalDescription = dr.description;
-                  apiRequest.dataContext.Items.push(item);    
+                  dataContext.Items.push(item);    
                 }
                 else
                   throw new Error(`Error running query ${queryName}`);
@@ -415,7 +473,7 @@ export class AskSkipResolver {
           apiRequest.requestPhase = 'data_gathering_response';
         }
       // we have all of the data now, add it to the data context and then submit it back to the Skip API
-      return this.HandleSkipRequest(apiRequest, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity);
+      return this.HandleSkipRequest(apiRequest, UserQuestion, user, dataSource, ConversationId, userPayload, pubSub, md, convoEntity, convoDetailEntity, dataContext, dataContextEntity);
     }
     catch (e) {
       LogError(e);
@@ -424,7 +482,7 @@ export class AskSkipResolver {
   }
 
   /**
-   * This method will handle the process for an end of request where a user is notified of an AI message. The AI message is either the finished report or a clarifying question.
+   * This method will handle the process for an end of successful request where a user is notified of an AI message. The AI message is either the finished report or a clarifying question.
    * @param apiResponse 
    * @param md 
    * @param user 
@@ -433,11 +491,11 @@ export class AskSkipResolver {
    * @param userPayload 
    * @returns 
    */
-  protected async FinishConversationAndNotifyUser(apiResponse: SkipAPIAnalysisCompleteResponse, md: Metadata, user: UserInfo, convoEntity: ConversationEntity, pubSub: PubSubEngine, userPayload: UserPayload): Promise<{AIMessageConversationDetailID: number}> {
+  protected async FinishConversationAndNotifyUser(apiResponse: SkipAPIAnalysisCompleteResponse, dataContext: SkipDataContext, dataContextEntity: DataContextEntity, md: Metadata, user: UserInfo, convoEntity: ConversationEntity, pubSub: PubSubEngine, userPayload: UserPayload): Promise<{AIMessageConversationDetailID: number}> {
     const sTitle = apiResponse.reportTitle; 
     const sResult = JSON.stringify(apiResponse);
 
-    // now, create a conversation detail record for the Skip response
+    // Create a conversation detail record for the Skip response
     const convoDetailEntityAI = <ConversationDetailEntity>await md.GetEntityObject('Conversation Details', user);
     convoDetailEntityAI.NewRecord();
     convoDetailEntityAI.ConversationID = convoEntity.ID;
@@ -447,7 +505,7 @@ export class AskSkipResolver {
     await convoDetailEntityAI.Save();
 
     // finally update the convo name if it is still the default
-    if (convoEntity.Name === AskSkipResolver._defaultNewChatName && sTitle) {
+    if (convoEntity.Name === AskSkipResolver._defaultNewChatName && sTitle && sTitle !== AskSkipResolver._defaultNewChatName) {
       convoEntity.Name = sTitle; // use the title from the response
       await convoEntity.Save();
     }
@@ -464,6 +522,24 @@ export class AskSkipResolver {
       conversationId: convoEntity.ID,
     });
     await userNotification.Save();
+    
+    // now, persist the data context items, first let's get 
+    for (const item of dataContext.Items) {
+      const dciEntity = <DataContextItemEntity>await md.GetEntityObject('Data Context Items', user);
+      if (item.DataContextItemID > 0) 
+        await dciEntity.Load(item.DataContextItemID);
+      else
+        dciEntity.NewRecord();
+      dciEntity.DataContextID = dataContextEntity.ID;
+      dciEntity.Type = item.Type;
+      dciEntity.RecordID = item.RecordID;
+      if (item.Type === 'sql')
+        dciEntity.SQL = item.RecordName; // the SQL field in the database is where we store the SQL, in the object model it ends up in the RecordName property, mapping it here
+      dciEntity.DataJSON = JSON.stringify(item.Data);
+      await dciEntity.Save();
+    }
+
+    // send a UI update trhough pub-sub
     pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
       message: JSON.stringify({
         type: 'UserNotifications',
@@ -475,6 +551,7 @@ export class AskSkipResolver {
       }),
       sessionId: userPayload.sessionId,
     });
+
     return {
       AIMessageConversationDetailID: convoDetailEntityAI.ID
     };