only_ever_generator 0.5.4 → 0.5.6

package/src/bootstrap/app.ts

@@ -7,142 +7,136 @@ import { GenerateTypology } from "../typology_gen/generate_typology";
 import { GenerateArgs } from "../utils/generate_args";
 import { gapFilling } from "../gap_fill/calculate_gap_fill";
 
-
 /// OnlyEverGenerator
 
 export class OnlyEverGenerator {
   public api_key: string = "";
   public openAiService: OpenAiService;
 
-/// these fields will be populated inside the constructor
+  /// these fields will be populated inside the constructor
   parsedContent: any = {};
   promptForTypology: string = "";
   promptForCardGen: string = "";
   expectedFields: Array<string>;
 
-
   typologyResponse: any = undefined;
   cardgenResponse: any = undefined;
   summarizeResponse = {};
   gapFillResponse: any = {};
 
-
-  constructor(
-    apiKey: string,
-    model: string,
-    generationContent : any
-  ) {
+  constructor(apiKey: string, model: string, generationContent: any) {
     this.api_key = apiKey;
     this.openAiService = new OpenAiService(
       apiKey,
       model ?? "gpt-3.5-turbo-1106"
     );
-    const parsedData = new ParseSourceContent(generationContent.content).parseData()
-    this.parsedContent = {
+    const parsedData = new ParseSourceContent(
+      generationContent.content
+    ).parseData();
+    (this.parsedContent = {
       title: parsedData.title,
       headings: parsedData.headings,
       content: parsedData.content,
       taxonomy: parsedData.taxonomy,
+    }),
+      // parsedData.type == 'cards' ? this.typologyResponse = parsedData.taxonomy :  this.typologyResponse = null;
+      (this.typologyResponse = generationContent.content.taxonomy);
 
-    },
-    // parsedData.type == 'cards' ? this.typologyResponse = parsedData.taxonomy :  this.typologyResponse = null;
-    this.typologyResponse = generationContent.content.taxonomy
-    
-    this.expectedFields =  generationContent.content.fields; //returnFields();
+    this.expectedFields = generationContent.content.fields; //returnFields();
     this.promptForTypology = generationContent.prompt.typology;
-    this.promptForCardGen =generationContent.prompt.card_generation;
+    this.promptForCardGen = generationContent.prompt.card_generation;
   }
 
-
- 
   async generate(
     generate_typology: boolean = false,
     generate_card: boolean = false
   ): Promise<Array<any>> {
-    let args = new GenerateArgs(generate_card, generate_typology, false, );
+    let args = new GenerateArgs(generate_card, generate_typology, false);
     const responseToReturn = [];
     const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
     for (let elem of whatNeedsToBeGenerated)
       if (elem == "generate_tyopology") {
         this.typologyResponse = await this.generateTypology(
-        this.promptForTypology
+          this.promptForTypology
         );
         responseToReturn.push(this.typologyResponse);
       } else if (elem == "generate_card") {
         /// for cards gen to occur, there must be presence of source taxonomy
-        if(this.shouldTheCardBeGeneratedAfterTypologyResponse()){
+        if (this.shouldTheCardBeGeneratedAfterTypologyResponse()) {
           this.parsedContent.taxonomy = {
             concepts: this.typologyResponse.concepts,
             facts: this.typologyResponse.facts,
             generate_cards: this.typologyResponse.generate_cards,
           };
-          this.cardgenResponse =  await this.generateCard(
+          this.cardgenResponse = await this.generateCard(
             this.promptForCardGen,
             JSON.stringify(this.typologyResponse),
-            false,
-          )
+            false
+          );
           responseToReturn.push(this.cardgenResponse);
 
-          /// check if gap fill is required ie coverage determination 
-          if(this.cardgenResponse.status_code == 200) {
-            this.gapFillResponse = await this._generationForGapFill(this.typologyResponse, this.cardgenResponse);
+          /// check if gap fill is required ie coverage determination
+          if (this.cardgenResponse.status_code == 200) {
+            this.gapFillResponse = await this._generationForGapFill(
+              this.typologyResponse,
+              this.cardgenResponse
+            );
             responseToReturn.push(this.gapFillResponse);
           }
-
         }
-    }
+      }
     return responseToReturn;
-   // return [typologyPrompt, cardPrompt];
-  
+    // return [typologyPrompt, cardPrompt];
   }
 
-  shouldTheCardBeGeneratedAfterTypologyResponse(){
-    if(this.typologyResponse){
+  shouldTheCardBeGeneratedAfterTypologyResponse() {
+    if (this.typologyResponse) {
       return this.typologyResponse?.generate_cards?.state == true;
-    }else{
+    } else {
       return false;
     }
-
   }
 
   async _generationForGapFill(typologyData: any, cardGenData: any) {
-      let gapFill = gapFilling(typologyData, cardGenData);
-      let response :any ;
-      if (
-        gapFill.remainingConcepts.length !== 0 ||
-        gapFill.remainingFacts.length !== 0
-      ) {
-        this.typologyResponse.facts = gapFill.remainingFacts;
-        this.typologyResponse.concepts = gapFill.remainingConcepts;
-        response = await this.generateCard(
-         this.promptForCardGen +
-            "Generate cards only suitable for the given remaining concepts and facts" +
-            JSON.stringify(gapFill) ,
-          
-            "",
-          true
-        );
-      }
-      return response;
-    
-  }
-
+    let gapFill = gapFilling(typologyData, cardGenData);
+    let response: any;
+    if (
+      gapFill.remainingConcepts.length !== 0 ||
+      gapFill.remainingFacts.length !== 0
+    ) {
+      this.typologyResponse.facts = gapFill.remainingFacts;
+      this.typologyResponse.concepts = gapFill.remainingConcepts;
+      response = await this.generateCard(
+        this.promptForCardGen +
+          "Generate cards only suitable for the given remaining concepts and facts" +
+          JSON.stringify(gapFill),
 
+        "",
+        true
+      );
+    }
+    return response;
+  }
 
-  async generateCard(prompt: string, additionalContent: string, isGapFill: boolean) {
-    let generateCardsResp = await new GenerateCards(this.openAiService).generateCards(
+  async generateCard(
+    prompt: string,
+    additionalContent: string,
+    isGapFill: boolean
+  ) {
+    let generateCardsResp = await new GenerateCards(
+      this.openAiService
+    ).generateCards(
       prompt ?? "",
       JSON.stringify(this.parsedContent) + additionalContent,
       isGapFill,
-      this.parsedContent.taxonomy,
+      this.parsedContent.taxonomy
     );
-  
+
     // let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
     // response['type'] = 'card_gen';
     return generateCardsResp;
   }
 
-
   async generateTypology(prompt: string) {
     let response = await new GenerateTypology(
       this.openAiService,
@@ -154,28 +148,24 @@ export class OnlyEverGenerator {
   }
 
   async gapFill(factsMaps: any, aiCards: Array<any>) {
-    /// factsmap 
+    /// factsmap
     /// {
     /// remaining_facts: [],
-     /// remaining_concepts: [],
+    /// remaining_concepts: [],
     //}
 
     /// aicards is data
-    let response :any ;
+    let response: any;
+
+    response = await this.generateCard(
+      this.promptForCardGen +
+        "Generate cards only suitable for the given remaining concepts and facts" +
+        JSON.stringify(factsMaps) +
+        "Exclude generating  cards with content in the following",
+      JSON.stringify(aiCards),
+      true
+    );
 
-      response = await this.generateCard(
-       this.promptForCardGen +
-          "Generate cards only suitable for the given remaining concepts and facts" +
-          JSON.stringify(factsMaps) +
-          "Exclude generating  cards with content in the following",
-          JSON.stringify(aiCards),
-        true
-      );
-    
     return response;
-  
-}
-
-  
-
+  }
 }

package/src/card_gen/generate_cards.ts

@@ -22,11 +22,16 @@ export class GenerateCards {
       req_type: response.type,
       req_tokens: response.usage_data?.prompt_tokens,
       res_tokens: response.usage_data?.completion_tokens,
+      prompt_tokens_details: response.usage_data?.prompt_tokens_details,
       model: this.openAiService.model,
     };
     if (response.status_code == 200) {
       response.metadata.status = "completed";
-      let parseCard = new ParseCardResponse().parse(response,isGapFill, taxonomy);
+      let parseCard = new ParseCardResponse().parse(
+        response,
+        isGapFill,
+        taxonomy
+      );
       return parseCard;
     } else {
       response.metadata.status = "failed";

package/src/config.ts

@@ -1,7 +1,7 @@
 import "dotenv/config";
 
 const config = {
-    openAIKey  : process.env.OPEN_AI_KEY as string
-}
+  openAIKey: process.env.OPEN_AI_KEY as string,
+};
 
-export default config;
+export default config;

package/src/constants/api_constants.ts

@@ -1,3 +1,3 @@
-export function openAiEndPoint(){
-    return 'https://api.openai.com/v1/chat/completions';
-}
+export function openAiEndPoint() {
+  return "https://api.openai.com/v1/chat/completions";
+}

package/src/constants/prompt_data.ts

@@ -2,8 +2,8 @@ import { returnCardGenPrompt } from "./prompts/card_gen_prompt";
 import { returnTypologyPrompt } from "./prompts/typology_prompt";
 
 const promptData: any = {
-    typology: {
-        role: `
+  typology: {
+    role: `
     As a dedicated assistant at a learning company, your role involves analyzing educational content to categorize and summarize it. You will process content (in JSON format) that represents text and images from diverse sources such as PDFs, book chapters, videos, and websites. Follow these steps:
     
     1. Classify the content into one to three predefined fields of knowledge.
@@ -27,7 +27,7 @@ const promptData: any = {
     
     Further instruction on how to perform these tasks are below.
         `,
-        fields: `
+    fields: `
         Every source must be placed under a field. This is the broadest category of knowledge. A source should belong to at least one and at most 3 fields. Only include fields that a source is strongly associated with. The field names in your response must exactly match the names of 18 fields listed below.
     
     1.	Sciences: Focus on Biology, Chemistry, Physics, Astronomy, Mathematics, and Computer Science.
@@ -49,7 +49,7 @@ const promptData: any = {
     17.	Reference & Indexing: Include Summaries, Timelines, Directories, Glossaries, Bibliographies, and other Reference Material.
     18.	Other: Use for content that doesn’t fit into the above categories.
         `,
-        concepts: `
+    concepts: `
         Extract key concepts within the content after classifying the field. Please be as exhaustive as possible.
     
     Definition of a Concept: Concepts are fundamental ideas that form the basis of knowledge in any discipline. They help organize and explain information, making it accessible and relatable.
@@ -66,7 +66,7 @@ const promptData: any = {
      ]
     }
         `,
-        facts: `
+    facts: `
         After classifying the content and identifying key concepts, proceed to extract and list verifiable facts.
     
     Definition of a Fact: Ensure each fact is a standalone piece of information that is concrete and can be independently verified.
@@ -77,7 +77,7 @@ const promptData: any = {
      "facts": ["fact1", "fact2", "fact3", "..."]
     }
         `,
-        generate: `
+    generate: `
         After analyzing the content, classifying its field, and identifying key concepts, and facts, assess whether the discovered elements warrant the creation of testing materials. Consider if these elements provide significant educational value to an average learner by enhancing understanding, offering practical applications, or supporting crucial educational goals. If you decide that testing cards don't need to be generated then please provide a reason in less than 90 characters.
     
     Value Assessment: Determine if the concepts and facts are essential for understanding the broader topic, are likely to be used in practical scenarios, or help in achieving educational benchmarks.
@@ -89,7 +89,7 @@ const promptData: any = {
          false_reason: "reason for marking the source as false. Leave empty for true."
      }
         `,
-        summarize:`
+    summarize: `
         After analyzing the content, identifying key concepts, and facts, summarize the material using a series of engaging and informative cards. These cards should capture the essence of the content while highlighting the critical concepts and facts that you previously identified.
     
     Inclusion Criteria: The generate_cards should be true. Return an empty array if the generate_cards is false.
@@ -102,10 +102,9 @@ const promptData: any = {
      "summary_cards": ["summary_card1", "summary_card2", "summary_card3", "..."]
     }
         `,
-        
-      },
-      card_generation:  {
-        role: `
+  },
+  card_generation: {
+    role: `
         As a dedicated assistant at a learning company, your role is to analyze educational content and create test cards that help learners understand and remember key concepts and facts. You will be provided with:
     
     1. Title of the source
@@ -146,7 +145,7 @@ const promptData: any = {
     
     Further instructions are provided below.
         `,
-        concepts : `
+    concepts: `
         You are provided with a list of identified concepts. Review this list and the content to determine if any concepts are missing.
     
     1. **Definition of a Concept**: Concepts are fundamental ideas that form the basis of knowledge in any discipline. They help organize and explain information, making it accessible and relatable.
@@ -163,7 +162,7 @@ const promptData: any = {
      ]
     }
         `,
-        facts :`
+    facts: `
         You are provided with a list of identified facts. Review this list and the content to determine if any facts are missing.
     
     1. **Definition of a Fact**: Standalone information that is concrete and independently verifiable.
@@ -174,7 +173,7 @@ const promptData: any = {
      "missing_facts": ["fact1", "fact2", "fact3", "..."]
     }
         `,
-        card_gen: `
+    card_gen: `
         After you have the complete list of concepts and facts, including any missing ones you identified, proceed to generate test cards for each.
     
      1.	Clarity: Ensure the test content is clear and unambiguous.
@@ -277,8 +276,8 @@ const promptData: any = {
     
     •	Maximum character length for each item in a pair: 42
         `,
-        reference : `Each test card needs a reference. A reference can either be the entire source or a specific heading in the source. Whenever possible, pick a main heading to direct the user to the most relevant part of the source material. The reference schema is as follows: source_title#main_heading, where #main_heading is optional.`,
-        checkcoverage: `Once you are done generating the test cards. Go back and evaulate the full list of concepts and fact that include any of the missing concepts or facts along with the list that was provided as the input. 
+    reference: `Each test card needs a reference. A reference can either be the entire source or a specific heading in the source. Whenever possible, pick a main heading to direct the user to the most relevant part of the source material. The reference schema is as follows: source_title#main_heading, where #main_heading is optional.`,
+    checkcoverage: `Once you are done generating the test cards. Go back and evaulate the full list of concepts and fact that include any of the missing concepts or facts along with the list that was provided as the input. 
     
     Are there any concept or fact that don't have a test card yet? If yes, go back and create one.
     
@@ -290,14 +289,13 @@ const promptData: any = {
      2.	If any concept or fact is missing a test card, create one for it.
      3.	Repeat this step until all concepts and facts are covered.
     
-    Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.`
-        
-      }
-}
+    Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.`,
+  },
+};
 
-export function returnPromptData(){
-    return {
-        "typology": returnTypologyPrompt(),
-        "card_generation": returnCardGenPrompt(),
-    };
-}
+export function returnPromptData() {
+  return {
+    typology: returnTypologyPrompt(),
+    card_generation: returnCardGenPrompt(),
+  };
+}

package/src/constants/prompts/card_gen_prompt.ts

@@ -152,8 +152,6 @@ Once you are done generating the test cards, review the full list of concepts an
 
 Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.
 `;
-export function returnCardGenPrompt(){
-    return promptString;
+export function returnCardGenPrompt() {
+  return promptString;
 }
-
-