only_ever_generator 1.0.7 → 1.0.8

package/src/index.ts

@@ -1,113 +1,7 @@
-// import express from "express";
-// import {
-//   returnCardResponse,
-//   returnHeadings,
-//   returnSourceData,
-// } from "./constants/source_data";
-// import config from "./config";
-// const app = express();
-// const port = 3000;
-// import { returnPromptData } from "./constants/prompt_data";
-// import { GenerateCards } from "./card_gen/generate_cards";
-// import { OpenAiService } from "./services/open_ai_service";
-// import { ParseCardResponse } from "./parse/parse_card_response";
-// import { returnTypologyPrompt } from "./constants/prompts/typology_prompt";
-// import { returnCardGenPrompt } from "./constants/prompts/card_gen_prompt";
-// import { GenerateArgs } from "./utils/generate_args";
-
 import { OnlyEverGenerator } from "./bootstrap/app";
-// import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
-// import { embeddingsResp } from "./typology-parsed-response";
-// import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
-// import { ParseEmbeddingResponse } from "./embedding_generation/parse_embedding_response";
+import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
+import { embeddingsResp } from "./typology-parsed-response";
+import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
+import { ParseEmbeddingResponse } from "./embedding_generation/parse_embedding_response";
 
-/// While Publishing the package , and using this code as a separate npm module
-/// uncomment the below line and comment all the others, expect the import of OnlyEverGenerator
 export { OnlyEverGenerator };
-
-// . All the Codes Below uses express and are strictly for development purpose, while publishing the package, comment everything
-// below this line
-// let oeGen = new OnlyEverGenerator(config.openAIKey, "gpt-4o-mini", {
-//   prompt: returnPromptData(),
-//   content: returnSourceData(),
-// });
-// app.get("/", async (req, res) => {
-//   // let data = oeGen.returnParsedContent();
-//   // let parsedData = parseResponse()
-//   let cardPrompt = "";
-
-//   res.send(cardPrompt);
-// });
-
-// app.get("/openAI", async (req, res) => {
-//   // let prompt = returnPromt();
-//   // let prompt = returnCardGenPrompt();
-//   //   let content = returnSourceData().toString()
-//   //   let headings = returnHeadings();
-//   //   // let aiRequest = await openAIRequest(prompt,content);
-//   let aiRequest = await oeGen.generate(true, true);
-//   res.send(aiRequest);
-// });
-
-// app.get("/parseCardData", async (req, res) => {
-//   let cardResp = returnCardResponse();
-//   let headings = returnHeadings();
-
-//   cardResp.metadata = {
-//     req_time: cardResp.generated_at ?? new Date(),
-//     req_type: "card",
-//     req_tokens: cardResp.usage_data?.prompt_tokens,
-//     res_tokens: cardResp.usage_data?.completion_tokens,
-//     model: "40-mini",
-//   };
-//   let parsedData = new ParseCardResponse().parse(cardResp, false, {});
-//   res.send(parsedData);
-// });
-
-// app.get("/generate-cards", async (req, res) => {
-//   let cardResp = embeddingsResp;
-//   oeGen.parsedContent.taxonomy = cardResp;
-//   let parsedData = await oeGen.generateCard(
-//     returnCardGenPrompt(),
-//     JSON.stringify(cardResp),
-//     false
-//   );
-//   res.send(parsedData);
-// });
-
-// app.get("/typology", async (req, res) => {
-//   {
-//     let typologyRequest = await oeGen.generate(true, false);
-//     res.send(typologyRequest);
-//   }
-// });
-
-// app.get("/local-consolidation", async (req, res) => {
-//   const concepts_facts = embeddingsResp.concepts_facts;
-
-//   let locallyConsolidated = new LocalConsolidation().consolidate(
-//     concepts_facts,
-//     "sourceID"
-//   );
-//   res.send(locallyConsolidated);
-// });
-
-// app.get("/global-consolidation", async (req, res) => {
-//   const concepts_facts = embeddingsResp.concepts_facts;
-
-//   // let globallyConsolidated = await new GlobalConsolidation().consolidate(
-//   //   concepts_facts,
-//   //   "sourceID",
-//   //   0.2
-//   // );
-//   // let globallyConsolidated = await oeGen.globalConsolidation(
-//   //   concepts_facts,
-//   //   "sourceID",
-//   //   0.2
-//   // );
-//   // res.send(globallyConsolidated);
-// });
-
-// app.listen(port, () => {
-//   console.log(`Example app listening at http://localhost:${port}`);
-// });

package/src/parse/parse_card/parse_cloze_card.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawClozeCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseClozeCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawClozeCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       const content = data.card_content;
       let correctOptions = content.correct_options;
@@ -36,9 +44,9 @@ export class ParseClozeCard {
           question: finalQuestion,
           options: finalParsedOptions,
         },
-        concepts: data.concepts,
-        facts: data.facts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
+        bloom_level: data.bloom_level,
       };
 
       return this._validateCloze(clozeCardData);

package/src/parse/parse_card/parse_flash_cards.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawFlashCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseFlashCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawFlashCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let displayTitle = this.generateFlashCardDisplayTitle(
         data.card_content.front,
@@ -8,7 +16,7 @@ export class ParseFlashCard {
       let flashCardData = {
         type: {
           category: "learning",
-          sub_type: data.type,
+          sub_type: "flash",
         },
         heading: "",
         displayTitle: displayTitle,
@@ -16,9 +24,9 @@ export class ParseFlashCard {
           front_content: data.card_content.front,
           back_content: data.card_content.back,
         },
-        concepts: data.concepts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
-        facts: data.facts,
+        bloom_level: data.bloom_level,
       };
 
       return flashCardData;

package/src/parse/parse_card/parse_match_card.ts

@@ -1,4 +1,6 @@
 import { match } from "assert";
+import { RawMatchCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
 
 type InputItem = {
   left_item: string;
@@ -11,7 +13,12 @@ type OutputItem = {
 };
 
 export class ParseMatchCard {
-  parse(cardData: any) {
+  parse(cardData: {
+    card_content: RawMatchCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let content = cardData.card_content;
       const finalContent = this._parseMatchContent(content);
@@ -26,9 +33,8 @@ export class ParseMatchCard {
         content: finalContent,
         //  content: cardData.card_content,
         displayTitle: displayTitle,
-        concepts: cardData.concepts,
-        facts: cardData.facts,
-        explanation: cardData.card_content.explanation,
+        concepts_facts: cardData.concepts_facts,
+        bloom_level: cardData.bloom_level,
       };
 
       return this._validateMatch(matchCard);

package/src/parse/parse_card/parse_mcq_card.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawMcqCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseMcqCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawMcqCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let mcqAnswers = [];
       if (
@@ -30,9 +38,9 @@ export class ParseMcqCard {
           question: data.card_content.prompt,
           answers: mcqAnswers,
         },
-        concepts: data.concepts,
-        facts: data.facts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
+        bloom_level: data.bloom_level,
       };
       //   return mcqCard;
       const isValid = this._validate(mcqCard);

package/src/parse/parse_card_response.ts

@@ -3,24 +3,60 @@ import { ParseClozeCard } from "./parse_card/parse_cloze_card";
 import { ParseFlashCard } from "./parse_card/parse_flash_cards";
 import { ParseMatchCard } from "./parse_card/parse_match_card";
 import { ParseMcqCard } from "./parse_card/parse_mcq_card";
-
+import {
+  GeneratedCardResponseType,
+  RawClozeCardResponseType,
+  RawFlashCardResponseType,
+  RawMatchCardResponseType,
+  RawMcqCardResponseType,
+  RawTestCardResponseType,
+} from "../types/raw_card_response_types/generated_card_response_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+import {
+  MongoConceptFactCards,
+  MongoConceptFactSource,
+} from "../types/mongo_concept_fact_type";
+import { ParsedCardType } from "../types/parsed_card_type";
 export class ParseCardResponse {
-  parse(generatedData: any, isGapFill: boolean, sourceTaxonomy: any) {
+  async parse(
+    generatedData: GeneratedCardResponseType,
+    isGapFill: boolean,
+    sourceTaxonomy: SourceTaxonomy,
+    bloom_level: number
+  ) {
     let usage_data = generatedData.metadata;
     try {
-      const cardData = [];
+      const cardData: ParsedCardType[] = [];
       const unparsedTestCards = generatedData.generated_content.test_cards;
-      const type = generatedData.type;
 
       if (unparsedTestCards !== undefined && unparsedTestCards.length != 0) {
         for (let elem of unparsedTestCards) {
+          elem.bloom_level = bloom_level;
+          const concepts = (elem.concepts ?? []).map((e: any) => {
+            return {
+              text: e,
+              type: "concept",
+            };
+          });
+          const facts = (elem.facts ?? []).map((e: any) => {
+            return {
+              text: e,
+              type: "fact",
+            };
+          });
+          const concepts_facts = [...concepts, ...facts];
           const managedCardConcepts = this._mapIdToConcepts(
-            elem.concepts_facts,
+            concepts_facts,
             sourceTaxonomy.concepts_facts
           );
-          elem.concepts = managedCardConcepts;
+
           if (elem.type == "flash") {
-            const flashCard = new ParseFlashCard().parse(elem);
+            const flashCard = new ParseFlashCard().parse({
+              card_content: elem.card_content as RawFlashCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (flashCard != null && flashCard) {
               flashCard.heading = this._getCardReference(
                 flashCard,
@@ -30,13 +66,23 @@ export class ParseCardResponse {
               cardData.push(flashCard);
             }
           } else if (elem.type == "mcq") {
-            const mcqCard = new ParseMcqCard().parse(elem);
+            const mcqCard = new ParseMcqCard().parse({
+              card_content: elem.card_content as RawMcqCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (mcqCard != null && mcqCard) {
               mcqCard.heading = this._getCardReference(mcqCard, sourceTaxonomy);
               cardData.push(mcqCard);
             }
           } else if (elem.type == "cloze") {
-            const clozeCard = new ParseClozeCard().parse(elem);
+            const clozeCard = new ParseClozeCard().parse({
+              card_content: elem.card_content as RawClozeCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (clozeCard && clozeCard != null) {
               clozeCard.heading = this._getCardReference(
                 clozeCard,
@@ -45,7 +91,12 @@ export class ParseCardResponse {
               cardData.push(clozeCard);
             }
           } else if (elem.type == "match") {
-            const matchCard = new ParseMatchCard().parse(elem);
+            const matchCard = new ParseMatchCard().parse({
+              card_content: elem.card_content as RawMatchCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (matchCard && matchCard != null) {
               matchCard.heading = this._getCardReference(
                 matchCard,
@@ -67,27 +118,46 @@ export class ParseCardResponse {
       return {
         status_code: cardData.length == 0 ? 400 : 200,
         metadata: usage_data,
-        type: type,
+        type: "card_gen",
 
         cards_data: cardData,
       };
     } catch (e: any) {
+      await new ErrorLogger({
+        type: "card_parsing",
+        data: {
+          error: e.message,
+          generatedData: generatedData,
+          sourceTaxonomy: sourceTaxonomy,
+        },
+      }).log();
       return {
         status_code: 500,
         metadata: usage_data,
-        type: generatedData.type,
+        type: "card_gen",
       };
     }
   }
 
-  _getCardReference(generatedCardData: any, sourceTaxonomy: any) {
-    const cardConcepts = (generatedCardData.concepts ?? []).map(
-      (e: any) => e.text
-    );
+  _parseCard(generatedCardData: any, sourceTaxonomy: any) {
+    const cardData: ParsedCardType[] = [];
+    const concepts = (generatedCardData.concepts ?? []).map((e: any) => {
+      return {
+        text: e,
+        type: "concept",
+      };
+    });
+  }
+  _getCardReference(card: any, sourceTaxonomy: any) {
+    const cardConcepts = (card.concepts_facts ?? []) as MongoConceptFactCards[];
+    const sourceConceptsFacts = (sourceTaxonomy.concepts_facts ??
+      []) as MongoConceptFactSource[];
+    if (cardConcepts.length == 0 || sourceConceptsFacts.length == 0) {
+      return "";
+    }
 
-    const sourceConceptsFacts = sourceTaxonomy.concepts_facts ?? [];
-    const firstMatchedConcept = sourceConceptsFacts.find((elem: any) =>
-      cardConcepts.includes(elem.text)
+    const firstMatchedConcept = sourceConceptsFacts.find(
+      (e: any) => cardConcepts[0].id == e.id
     );
 
     if (firstMatchedConcept) {
@@ -97,7 +167,10 @@ export class ParseCardResponse {
     }
   }
 
-  _mapIdToConcepts(cardConcepts: any[], sourceConceptsFacts: any[]) {
+  _mapIdToConcepts(
+    cardConcepts: any[],
+    sourceConceptsFacts: any[]
+  ): MongoConceptFactCards[] {
     // const cardData = cardC
     const managedCardConcepts = [];
     for (const cardConcept of cardConcepts) {
@@ -109,6 +182,7 @@ export class ParseCardResponse {
         managedCardConcepts.push({
           id: matchedConcept.id,
           text: matchedConcept.text,
+          type: matchedConcept.type,
         });
       }
     }

package/src/parse/parse_source_content.ts

@@ -1,3 +1,5 @@
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+
 export class ParseSourceContent {
   public content: any;
   /// Format of Content
@@ -25,7 +27,14 @@ export class ParseSourceContent {
     this.content = sourceContent;
   }
 
-  parseData() {
+  parseData(): {
+    source_id: string;
+    type: string;
+    title: string;
+    content: any[];
+    headings: string[];
+    taxonomy: SourceTaxonomy;
+  } {
     let sourceType = this.content.type;
     let afterSanitized;
     if (sourceType == "video") {
@@ -37,6 +46,7 @@ export class ParseSourceContent {
       afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
     }
     return {
+      source_id: this.content.source_id,
       type: this.content.type,
       title: this.content.title,
       content: afterSanitized,

package/src/types/base_param_type.ts

@@ -0,0 +1,17 @@
+import { SourceTaxonomy } from "./source_taxonomy_type";
+
+export type BaseParamType = {
+  prompt: {
+    typology: string;
+    card_generation: string;
+  };
+  content: {
+    source_id: string;
+    title: string;
+    headings: string[];
+    content: any[];
+    fields: string[];
+    taxonomy?: SourceTaxonomy;
+    type: string;
+  };
+};

package/src/types/mongo_concept_fact_type.ts

@@ -0,0 +1,12 @@
+export type MongoConceptFactSource = {
+  text: string;
+  type: "concept" | "fact";
+  reference: string;
+  id: string;
+};
+
+export type MongoConceptFactCards = {
+  text: string;
+  type: "concept" | "fact";
+  id: string;
+};

package/src/types/parsed_card_type.ts

@@ -0,0 +1,39 @@
+export type ParsedCardType = {
+  type: {
+    category: string;
+    sub_type: string;
+  };
+  heading: string;
+  displayTitle: string;
+  content: any;
+  explanation?: string;
+  bloom_level?: number;
+};
+
+// {
+//     "type": {
+//       "category": "learning",
+//       "sub_type": "cloze"
+//     },
+//     "heading": "",
+//     "displayTitle": "Ink is a medium that consists of colorants and other {{components}}. ---- components, elements, substances",
+//     "content": {
+//       "question": "Ink is a medium that consists of colorants and other {{c0:components}}.",
+//       "options": [
+//         {
+//           "option": "components",
+//           "cloze": "c0"
+//         },
+//         {
+//           "option": "elements",
+//           "cloze": "null"
+//         },
+//         {
+//           "option": "substances",
+//           "cloze": "null"
+//         }
+//       ]
+//     },
+//     "explanation": "Ink contains various components that include colorants, solvents, and additives which determine its properties.",
+//     "bloom_level": 1
+//   },

package/src/types/raw_card_response_types/generated_card_response_type.ts

@@ -0,0 +1,61 @@
+import { CardTypeEnum } from "../../enums/card_type_enum";
+
+export type GeneratedCardResponseType = {
+  metadata: any;
+  generated_content: {
+    test_cards: RawTestCardResponseType[];
+  };
+  status_code: number;
+  usage_data: any;
+  generated_at: Date;
+};
+
+export type RawTestCardResponseType = {
+  type: string;
+  // type:
+  //   | CardTypeEnum.FLASH
+  //   | CardTypeEnum.MATCH
+  //   | CardTypeEnum.MCQ
+  //   | CardTypeEnum.CLOZE;
+  card_content:
+    | RawFlashCardResponseType
+    | RawMatchCardResponseType
+    | RawClozeCardResponseType
+    | RawMcqCardResponseType;
+  concepts: string[];
+  facts: string[];
+  bloom_level: number;
+};
+
+export type RawFlashCardResponseType = {
+  front: string;
+  back: string;
+  explanation?: string;
+};
+
+export type RawMatchCardResponseType = {
+  left_item: string;
+  right_item: string;
+}[];
+
+export type RawClozeCardResponseType = {
+  correct_options: string[];
+  incorrect_options: string[];
+  prompt: string;
+  explanation?: string;
+};
+
+export type RawMcqCardResponseType = {
+  prompt: string;
+  explanation?: string;
+  choices: {
+    choice: string;
+    is_correct: boolean;
+  }[];
+};
+
+//
+// flash : 1
+// cloze 0,
+// match 2
+// mcq 3

package/src/types/source_taxonomy_type.ts

@@ -0,0 +1,17 @@
+export type SourceTaxonomy = {
+  concepts_facts: {
+    text: string;
+    type: string;
+    /// will only be present temporarily and wont be stored in database
+    embedding?: number[];
+    reference: string;
+  }[];
+  fields: string[];
+
+  /// will only be present temporarily and wont be stored in database
+  metadata?: any[];
+  generate_cards: {
+    state: boolean;
+    reason: string;
+  };
+};

package/src/typology_gen/generate_typology.ts

@@ -1,6 +1,5 @@
-import { OpenAiService } from "../services/open_ai_service";
-import { returnTypologyData } from "../parse/response_format_typology";
 import { ErrorLogger } from "../logger";
+import { OpenAiService } from "../services/open_ai_service";
 
 export class GenerateTypology {
   public openAiService: OpenAiService;

package/.env.example

@@ -1 +0,0 @@
-OPEN_AI_KEY= 'open ai key'

package/dist/constants/prompt_data.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompt_data.d.ts","sourceRoot":"","sources":["../../src/constants/prompt_data.ts"],"names":[],"mappings":"AAuSA,wBAAgB,gBAAgB;;;EAK/B"}

package/dist/constants/source_data.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"source_data.d.ts","sourceRoot":"","sources":["../../src/constants/source_data.ts"],"names":[],"mappings":"AAonBA,wBAAgB,YAAY,aAqB3B;AAWD,wBAAgB,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAE/B;AAED,wBAAgB,cAAc,aAE7B;AAED,wBAAgB,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAEjC"}

package/dist/parse/response_format_card.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"response_format_card.d.ts","sourceRoot":"","sources":["../../src/parse/response_format_card.ts"],"names":[],"mappings":"AAAA,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;CAkJjB,CAAC;AAEF,QAAA,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Of,CAAC"}

package/dist/parse/response_format_typology.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"response_format_typology.d.ts","sourceRoot":"","sources":["../../src/parse/response_format_typology.ts"],"names":[],"mappings":"AAyCA,wBAAgB,kBAAkB,IACL,GAAG,CAC/B"}

package/prompts.json

@@ -1,23 +0,0 @@
-[
-    {
-        "generation_for":"card",
-        "type": "role",
-        "prompt": "As a dedicated assistant at a learning company, your role is to create test cards based on the provided content. The purpose of these cards is to help learners master and understand the concepts and facts presented to you."
-    },
-    {
-        "generation_for":"card",
-        "type": "inputs",
-        "prompt": "You will be provided with the following: \n 1. Title of the source \n 2. The content \n 3. The field of knowledge it belongs to \n 4. Key concepts in the source \n 5. Important facts in the source"
-    },
-    {
-        "generation_for":"card",
-        "type" :"steps",
-        "prompt": "**Follow these steps:** \n 1. Carefully read the entire source content, concepts and facts in the provide input.\n  2. Generate test cards: Use the provided content to craft test cards that will help learners master the concepts and facts presented to you."
-    },
-    {
-        "generation_for": "card",
-        "type": "schema",
-        "prompt": "**Format your response in the following JSON format:** json { \"test_cards\": [ { \"type\": \"{card_type}\", \"card_content\": \"{content}\", \"concepts\": [\"concept1\", \"concept2\", \"...\"], \"facts\": [\"fact1\", \"fact2\", \"...\"] }, {... as many cards as possible} ] }"
-
-    }
-]