only_ever_generator 1.0.2 → 1.0.4

package/src/index.ts

@@ -1,179 +1,113 @@
-
-import { ParseSourceContent } from "./class/parse/parse_source_content";
-import { OpenAiService } from "./class/services/open_ai_service";
-import { returnCardGenPrompt } from "./constants/prompts/card_gen_prompt";
-import { returnTypologyPrompt } from "./constants/prompts/typology_prompt";
+// import express from "express";
+// import {
+//   returnCardResponse,
+//   returnHeadings,
+//   returnSourceData,
+// } from "./constants/source_data";
+// import config from "./config";
 // const app = express();
 // const port = 3000;
-// let openAiService = new OpenAiService(config.openAIKey);
-
-
+// import { returnPromptData } from "./constants/prompt_data";
+// import { GenerateCards } from "./card_gen/generate_cards";
+// import { OpenAiService } from "./services/open_ai_service";
+// import { ParseCardResponse } from "./parse/parse_card_response";
+// import { returnTypologyPrompt } from "./constants/prompts/typology_prompt";
+// import { returnCardGenPrompt } from "./constants/prompts/card_gen_prompt";
+// import { GenerateArgs } from "./utils/generate_args";
+
+import { OnlyEverGenerator } from "./bootstrap/app";
+// import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
+// import { embeddingsResp } from "./typology-parsed-response";
+// import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
+// import { ParseEmbeddingResponse } from "./embedding_generation/parse_embedding_response";
+
+/// While Publishing the package , and using this code as a separate npm module
+/// uncomment the below line and comment all the others, expect the import of OnlyEverGenerator
+export { OnlyEverGenerator };
+
+// . All the Codes Below uses express and are strictly for development purpose, while publishing the package, comment everything
+// below this line
+// let oeGen = new OnlyEverGenerator(config.openAIKey, "gpt-4o-mini", {
+//   prompt: returnPromptData(),
+//   content: returnSourceData(),
+// });
+// app.get("/", async (req, res) => {
+//   // let data = oeGen.returnParsedContent();
+//   // let parsedData = parseResponse()
+//   let cardPrompt = "";
 
-// let oeGen =  new OnlyEverGenerator(config.openAIKey, returnSourceData())
-// app.get('/', async (req, res)  => {
-//     let data = oeGen.returnParsedContent();
-//     // let parsedData = parseResponse()
-//     res.send(data);
+//   res.send(cardPrompt);
 // });
 
-// app.get('/openAI', async (req,res)=> {
+// app.get("/openAI", async (req, res) => {
 //   // let prompt = returnPromt();
-  
-//   let prompt = returnCardGenPrompt();
-//     let content = returnSourceData().toString()
-//     let headings = returnHeadings();
-//     // let aiRequest = await openAIRequest(prompt,content);
-//     let aiRequest = await oeGen.generateCard(prompt,content);
-//     res.send(aiRequest);
-// }); 
-
-// app.get('/typology', async(req,res)=>{
-//   { 
-//     let typologyPrompt = returnTypologyPrompt();
-//     let cardPrompt = returnCardGenPrompt();
-//     let args = new GenerateArgs(
-//       true,
-//       true,
-//       false,
-//        {
-//         typology_prompt: typologyPrompt,
-//         card_gen_prompt: cardPrompt,
-//         summary_prompt: "",
-//       }
-//     )
-//     let typologyRequest = await oeGen.generate(
-//       args,
-//     );
-//     res.send(typologyRequest);
-  
-//   }
+//   // let prompt = returnCardGenPrompt();
+//   //   let content = returnSourceData().toString()
+//   //   let headings = returnHeadings();
+//   //   // let aiRequest = await openAIRequest(prompt,content);
+//   let aiRequest = await oeGen.generate(true, true);
+//   res.send(aiRequest);
 // });
 
-// app.listen(port, () => {
-//   console.log(`Example app listening at http://localhost:${port}`);
+// app.get("/parseCardData", async (req, res) => {
+//   let cardResp = returnCardResponse();
+//   let headings = returnHeadings();
+
+//   cardResp.metadata = {
+//     req_time: cardResp.generated_at ?? new Date(),
+//     req_type: "card",
+//     req_tokens: cardResp.usage_data?.prompt_tokens,
+//     res_tokens: cardResp.usage_data?.completion_tokens,
+//     model: "40-mini",
+//   };
+//   let parsedData = new ParseCardResponse().parse(cardResp, false, {});
+//   res.send(parsedData);
 // });
 
+// app.get("/generate-cards", async (req, res) => {
+//   let cardResp = embeddingsResp;
+//   oeGen.parsedContent.taxonomy = cardResp;
+//   let parsedData = await oeGen.generateCard(
+//     returnCardGenPrompt(),
+//     JSON.stringify(cardResp),
+//     false
+//   );
+//   res.send(parsedData);
+// });
 
+// app.get("/typology", async (req, res) => {
+//   {
+//     let typologyRequest = await oeGen.generate(true, false);
+//     res.send(typologyRequest);
+//   }
+// });
 
+// app.get("/local-consolidation", async (req, res) => {
+//   const concepts_facts = embeddingsResp.concepts_facts;
 
+//   let locallyConsolidated = new LocalConsolidation().consolidate(
+//     concepts_facts,
+//     "sourceID"
+//   );
+//   res.send(locallyConsolidated);
+// });
 
+// app.get("/global-consolidation", async (req, res) => {
+//   const concepts_facts = embeddingsResp.concepts_facts;
+
+//   // let globallyConsolidated = await new GlobalConsolidation().consolidate(
+//   //   concepts_facts,
+//   //   "sourceID",
+//   //   0.2
+//   // );
+//   // let globallyConsolidated = await oeGen.globalConsolidation(
+//   //   concepts_facts,
+//   //   "sourceID",
+//   //   0.2
+//   // );
+//   // res.send(globallyConsolidated);
+// });
 
-
-
-
-export class OnlyEverGenerator{
-    public api_key: string = '';
-    public openAiService: OpenAiService | undefined;
-    parsedContent: String = '';
-    constructor(apiKey:string, model: String, content: Array<any>){
-        this.api_key = apiKey;
-        this.openAiService = new OpenAiService(apiKey,model ?? 'gpt-3.5-turbo-1106');
-        this.parsedContent = new ParseSourceContent(content).parse();
-    };
-
-    typologyResponse = {};
-    cardgenResponse = {};
-    summarizeResponse = {};
-
-    
-    
-
-    async generate( 
-       generate_card : boolean = false,
-       generate_typology: boolean =false,
-    ): Promise<Array<any>> {
-      let typologyPrompt = returnTypologyPrompt();
-      let cardPrompt = returnCardGenPrompt();
-      let args = new GenerateArgs(
-        generate_card,
-        generate_typology,
-        false,
-        {
-          typology_prompt: typologyPrompt,
-          card_gen_prompt: cardPrompt,
-          summary_prompt: ''
-        },
-        
-      );
-        const responseToReturn = [];
-        const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
-        for(let elem of whatNeedsToBeGenerated)
-        if(elem == 'generate_tyopology'){
-            this.typologyResponse = await this.generateTypology(args.prompts.typology_prompt ?? '', this.parsedContent);
-            responseToReturn.push(this.typologyResponse);
-        }else if(elem == 'generate_card'){
-
-            this.cardgenResponse =  await this.generateCard(args.prompts.card_gen_prompt ?? '', this.parsedContent +  JSON.stringify(this.typologyResponse));
-            responseToReturn.push(this.cardgenResponse);
-        }else if(elem == 'generate_summary'){
-            this.summarizeResponse = await this.generateSummary(args.prompts.summary_prompt ?? '', this.parsedContent);
-            responseToReturn.push(this.summarizeResponse);
-        }
-
-        return responseToReturn;
-        
-    }
-    
-    _returnParsedContent(){
-        return this.parsedContent;
-    }
-
-
-    async generateCard(prompt: String, content: String){
-        let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-        response['type'] = 'card_gen';
-        return response;
-    }
-
-    async generateTypology(prompt:String,content:String){
-        let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-        response['type'] = 'typology';
-        return response;
-    }
-
-    async generateSummary(prompt:String,content:String){
-        let response = await this.openAiService?.sendRequest(prompt,this.parsedContent);
-        response['type']= 'summary';
-        return response;
-    }
-    
-}
-
-export class GenerateArgs{
-    public generate_card : boolean = false;
-    public generate_typology: boolean = false;
-    public generate_summary: boolean = false;
-    public prompts = {
-        typology_prompt: '',
-        card_gen_prompt: '',
-        summary_prompt: ''
-    }
-
-    constructor(generate_card: boolean,generate_typology: boolean, generate_summary: boolean, prompts = {
-        typology_prompt: '',
-        card_gen_prompt: '',
-        summary_prompt: ''
-    }){
-        this.generate_card = generate_card;
-        this.generate_typology = generate_typology;
-        this.generate_summary = generate_summary;
-        this.prompts = prompts
-    }
-
-    getWhatNeedsToBeGenerated(){
-        let returnData = [];
-        if(this.generate_typology == true){
-            returnData.push('generate_tyopology')
-        }
-        if(this.generate_summary == true){
-            returnData.push('generate_summary')
-        }
-        if(this.generate_card == true){
-            returnData.push('generate_card');
-        }
-        return returnData;
-        
-        
-    }
-
-}
+// app.listen(port, () => {
+//   console.log(`Example app listening at http://localhost:${port}`);
+// });

package/src/logger.ts

@@ -0,0 +1,30 @@
+/// this method will call out atlas function and will write to a doc, incase of any errors:
+/// this is only for developmemt use
+
+import axios from "axios";
+
+export class ErrorLogger {
+  data: any;
+  constructor(data: any) {
+    this.data = data;
+  }
+
+  async log() {
+    try {
+      let response = await axios.post(
+        "https://us-east-1.aws.data.mongodb-api.com/app/oe-phase1-tkmsy/endpoint/oe_gen_logger",
+        {
+          data: this.data,
+        },
+        {
+          headers: {
+            "Content-Type": ["application/json"],
+          },
+        }
+      );
+      return response;
+    } catch (e) {
+      console.log(e);
+    }
+  }
+}

package/src/parse/parse_card/parse_cloze_card.ts

@@ -0,0 +1,136 @@
+export class ParseClozeCard {
+  parse(data: any) {
+    try {
+      const content = data.card_content;
+      let correctOptions = content.correct_options;
+      let incorrectOptions = content.incorrect_options;
+      let allOptions = [...correctOptions, ...incorrectOptions];
+      let displayTitle = this._generateClozeCardDisplayTitle(
+        data.card_content.prompt,
+        allOptions
+      );
+      let preparedData = this._prepareQuestionAndCorrectAnswers(
+        content.prompt,
+        correctOptions
+      );
+      let finalQuestion = preparedData.prompt;
+      let parsedCorrectOptions = preparedData.options;
+      let parsedIncorrectoptions = incorrectOptions.map((e: any) => {
+        return {
+          option: e,
+          cloze: "null",
+        };
+      });
+      const finalParsedOptions = [
+        ...parsedCorrectOptions,
+        ...parsedIncorrectoptions,
+      ];
+      let clozeCardData = {
+        type: {
+          category: "learning",
+          sub_type: data.type,
+        },
+        heading: "",
+        displayTitle: displayTitle,
+        content: {
+          question: finalQuestion,
+          options: finalParsedOptions,
+        },
+        concepts: data.concepts,
+        facts: data.facts,
+        explanation: data.card_content.explanation,
+      };
+
+      return this._validateCloze(clozeCardData);
+    } catch (e) {
+      return null;
+    }
+  }
+
+  _generateClozeCardDisplayTitle(question: string, answers: Array<any>) {
+    try {
+      let optionsString = "";
+      if (answers.length !== 0) {
+        optionsString = answers.join(", ");
+      }
+
+      return `${question} ---- ${optionsString}`;
+    } catch (e) {
+      throw Error("Error in generating display title");
+    }
+  }
+
+  /// validate the cloze card
+  // 1. Has Empty cloze
+  // 2. has Duplicate Clozes
+  // 3. doesnt have any valid option,
+  // 4. Question length 320
+  // 5. More than 6 options
+  // 6. Less than 2 options
+  // 7. Max character for individual cloze: 90
+
+  _prepareQuestionAndCorrectAnswers(
+    rawPrompt: String,
+    correctOptions: Array<any>
+  ) {
+    try {
+      var finalCorrectOptions = <any>[];
+      const regex = /{{(.*?)}}/g;
+
+      const transformed = rawPrompt.replace(regex, (match, p1) => {
+        // p1 is the captured group inside {{ }} (e.g., "fruit", "green")
+        const idx = correctOptions.indexOf(p1);
+        if (idx !== -1) {
+          let cloze = `c${idx}`;
+          finalCorrectOptions.push({
+            option: p1,
+            cloze: cloze,
+          });
+          return `{{c${idx}:${p1}}}`;
+        }
+        return match; // If not found in correct_options, leave as is or handle accordingly
+      });
+      return {
+        prompt: transformed,
+        options: finalCorrectOptions,
+      };
+    } catch (e) {
+      throw Error("Error in preparing question and correct answers");
+    }
+  }
+
+  _validateCloze(clozeCard: any) {
+    let clozeRegex = /\{\{c(\d+):([^}]+)\}\}/g;
+
+    try {
+      /// validate emptu cloze
+      let options = clozeCard.content.options ?? [];
+      let question = clozeCard.content.question;
+      if (options.length < 2 || options.length > 6) {
+        throw Error("Number of cloze options are invalid");
+      }
+
+      /// There are no correct clozes// or null cloze or empty cloze
+      let correctOptions = options.find(
+        (e: any) => e.cloze != "null" && e.cloze != null && e.cloze.trim() != ""
+      );
+      if (correctOptions) {
+      } else {
+        throw Error(" No valid clozes exists");
+      }
+
+      let rightClozes = options.filter((e: any) => e.cloze.startsWith("c"));
+
+      /// matches our cloze syntax
+      let clozeMatches = [...question.matchAll(clozeRegex)];
+      if (clozeMatches.length == 0) {
+        throw Error("Question Invalid");
+      } else if (clozeMatches.length != rightClozes.length) {
+        throw Error(" Clozes in question doesnt match to clozes in options");
+      }
+      return clozeCard;
+    } catch (e: any) {
+      throw Error(`Error in validating cloze card ${e.message}`);
+    }
+  }
+}

package/src/parse/parse_card/parse_flash_cards.ts

@@ -0,0 +1,33 @@
+export class ParseFlashCard {
+  parse(data: any) {
+    try {
+      let displayTitle = this.generateFlashCardDisplayTitle(
+        data.card_content.front,
+        data.card_content.back
+      );
+      let flashCardData = {
+        type: {
+          category: "learning",
+          sub_type: data.type,
+        },
+        heading: "",
+        displayTitle: displayTitle,
+        content: {
+          front_content: data.card_content.front,
+          back_content: data.card_content.back,
+        },
+        concepts: data.concepts,
+        explanation: data.card_content.explanation,
+        facts: data.facts,
+      };
+
+      return flashCardData;
+    } catch (e) {
+      return null;
+    }
+  }
+
+  generateFlashCardDisplayTitle(front: string, back: string) {
+    return `${front} ---- ${back}`;
+  }
+}

package/src/parse/parse_card/parse_match_card.ts

@@ -0,0 +1,94 @@
+import { match } from "assert";
+
+type InputItem = {
+  left_item: string;
+  right_item: string;
+};
+
+type OutputItem = {
+  left_item: string;
+  right_item: string[];
+};
+
+export class ParseMatchCard {
+  parse(cardData: any) {
+    try {
+      let content = cardData.card_content;
+      const finalContent = this._parseMatchContent(content);
+
+      let displayTitle = this._generateMatchCardDisplayTitle(content);
+      let matchCard = {
+        type: {
+          category: "learning",
+          sub_type: cardData.type,
+        },
+        heading: "",
+        content: finalContent,
+        //  content: cardData.card_content,
+        displayTitle: displayTitle,
+        concepts: cardData.concepts,
+        facts: cardData.facts,
+        explanation: cardData.card_content.explanation,
+      };
+
+      return this._validateMatch(matchCard);
+    } catch (e) {
+      return null;
+    }
+  }
+
+  _generateMatchCardDisplayTitle(answers: any) {
+    let titles: string[] = [];
+    let counter = 65;
+    for (let data of answers) {
+      let value = data.right_item;
+      let leftData = data.left_item;
+      let letter = String.fromCharCode(counter);
+      titles.push(`${letter}. ${leftData} -- ${value}`);
+      counter++;
+    }
+    let displayTitle = titles.join(",");
+    return displayTitle;
+  }
+
+  _parseMatchContent = (input: InputItem[]): OutputItem[] => {
+    const grouped = input.reduce<Record<string, OutputItem>>(
+      (acc, { left_item, right_item }) => {
+        if (!acc[left_item]) {
+          acc[left_item] = { left_item, right_item: [] };
+        }
+        acc[left_item].right_item.push(right_item);
+        return acc;
+      },
+      {}
+    );
+
+    return Object.values(grouped);
+  };
+
+  _validateMatch(matchCard: any) {
+    let matches = matchCard.content;
+    let content = [];
+    try {
+      if (matches.length < 1 || matches.length > 8) {
+        throw Error("Invalid number of matches");
+      }
+
+      for (let elem of matches) {
+        if (elem.left_item.length <= 30 && elem.left_item.length != 0) {
+          if (elem.right_item.length <= 40 && elem.right_item.length != 0) {
+            content.push(elem);
+          }
+        }
+      }
+      if (content.length >= 2) {
+        matchCard.content = content;
+      } else {
+        throw Error("Invalid content");
+      }
+      return matchCard;
+    } catch (e) {
+      return null;
+    }
+  }
+}

package/src/parse/parse_card/parse_mcq_card.ts

@@ -0,0 +1,105 @@
+export class ParseMcqCard {
+  parse(data: any) {
+    try {
+      let mcqAnswers = [];
+      if (
+        data.card_content.choices !== undefined &&
+        data.card_content.choices.length != 0
+      ) {
+        for (let choice of data.card_content.choices) {
+          let answer = {
+            answer: choice.choice,
+            is_correct: choice.is_correct,
+          };
+          mcqAnswers.push(answer);
+        }
+      }
+
+      let displayTitle = this._generateMcqCardDisplayTitle(
+        data.card_content.prompt,
+        mcqAnswers
+      );
+      let mcqCard = {
+        type: {
+          category: "learning",
+          sub_type: data.type,
+        },
+        heading: "",
+        displayTitle: displayTitle,
+        content: {
+          question: data.card_content.prompt,
+          answers: mcqAnswers,
+        },
+        concepts: data.concepts,
+        facts: data.facts,
+        explanation: data.card_content.explanation,
+      };
+      //   return mcqCard;
+      const isValid = this._validate(mcqCard);
+      if (isValid == true) {
+        return mcqCard;
+      } else {
+        return null;
+      }
+    } catch (e) {
+      return null;
+    }
+  }
+
+  _generateMcqCardDisplayTitle(question: string, answers: any) {
+    let answersString = [];
+    if (answers.length != 0) {
+      for (let option of answers) {
+        let currentIndex = answers.indexOf(option) + 1;
+        let temp = `${currentIndex} . ${option.answer} `;
+        answersString.push(temp);
+      }
+      let resultString = answersString.join("");
+      let finalDisplayTitle = `${question} ---- ${resultString}`;
+      return finalDisplayTitle;
+    } else {
+      return question;
+    }
+  }
+
+  /// validate mcq card
+  // 1. Check if atleast 1 correct answer exists
+  // 2. Length of answer shouldnt exceed  24 chars
+  // 3. Length of question shouldnt exceed 90 chars
+  // 4. If Any option is Empty
+  _validate(mcqCard: any) {
+    try {
+      let isQuestionValid = mcqCard.content.question.length <= 90;
+      if (!isQuestionValid) {
+        throw new Error("Question length exceeded");
+      }
+
+      /// check if all are wrong answers
+      let exists = this._checkIfAllAnswersAreWrong(mcqCard.content.answers);
+      if (exists) {
+      } else {
+        throw new Error("Every answers are wrong");
+      }
+
+      /// check if answers are of length <40 or is 0
+      let answerWhoseLengthisGreaterThan40or0 = (
+        mcqCard.content.answers ?? []
+      ).find((e: any) => e.answer.length == 0 || e.answer.length > 40);
+      if (answerWhoseLengthisGreaterThan40or0) {
+        throw new Error("Option has length more than 40 or is Empty");
+      }
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  _checkIfAllAnswersAreWrong(answers: any[]) {
+    let rightAnswer = answers.find((e) => e.is_correct == true);
+    if (rightAnswer) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+}