only_ever_generator 0.7.8 → 0.8.0

package/src/parse/parse_card/parse_mcq_card.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawMcqCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseMcqCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawMcqCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let mcqAnswers = [];
       if (
@@ -30,9 +38,9 @@ export class ParseMcqCard {
           question: data.card_content.prompt,
           answers: mcqAnswers,
         },
-        concepts: data.concepts,
-        facts: data.facts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
+        bloom_level: data.bloom_level,
       };
       //   return mcqCard;
       const isValid = this._validate(mcqCard);

package/src/parse/parse_card_response.ts

@@ -3,33 +3,83 @@ import { ParseClozeCard } from "./parse_card/parse_cloze_card";
 import { ParseFlashCard } from "./parse_card/parse_flash_cards";
 import { ParseMatchCard } from "./parse_card/parse_match_card";
 import { ParseMcqCard } from "./parse_card/parse_mcq_card";
-
+import {
+  GeneratedCardResponseType,
+  RawClozeCardResponseType,
+  RawFlashCardResponseType,
+  RawMatchCardResponseType,
+  RawMcqCardResponseType,
+  RawTestCardResponseType,
+} from "../types/raw_card_response_types/generated_card_response_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+import { MongoConceptFactCards } from "../types/mongo_concept_fact_type";
+import { ParsedCardType } from "../types/parsed_card_type";
 export class ParseCardResponse {
-  parse(generatedData: any, isGapFill: boolean, sourceTaxonomy: any) {
+  parse(
+    generatedData: GeneratedCardResponseType,
+    isGapFill: boolean,
+    sourceTaxonomy: SourceTaxonomy,
+    bloom_level: number
+  ) {
     let usage_data = generatedData.metadata;
     try {
-      const cardData = [];
+      const cardData: ParsedCardType[] = [];
       const unparsedTestCards = generatedData.generated_content.test_cards;
-      const type = generatedData.type;
+
       if (unparsedTestCards !== undefined && unparsedTestCards.length != 0) {
         for (let elem of unparsedTestCards) {
+          elem.bloom_level = bloom_level;
+          const concepts = elem.concepts.map((e: any) => {
+            return {
+              text: e,
+              type: "concept",
+            };
+          });
+          const facts = elem.facts.map((e: any) => {
+            return {
+              text: e,
+              type: "fact",
+            };
+          });
+          const concepts_facts = [...concepts, ...facts];
+          const managedCardConcepts = this._mapIdToConcepts(
+            concepts_facts,
+            sourceTaxonomy.concepts_facts
+          );
+
           if (elem.type == "flash") {
-            const flashCard = new ParseFlashCard().parse(elem);
+            const flashCard = new ParseFlashCard().parse({
+              card_content: elem.card_content as RawFlashCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (flashCard != null && flashCard) {
               flashCard.heading = this._getCardReference(
                 flashCard,
                 sourceTaxonomy
               );
+
               cardData.push(flashCard);
             }
           } else if (elem.type == "mcq") {
-            const mcqCard = new ParseMcqCard().parse(elem);
+            const mcqCard = new ParseMcqCard().parse({
+              card_content: elem.card_content as RawMcqCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (mcqCard != null && mcqCard) {
               mcqCard.heading = this._getCardReference(mcqCard, sourceTaxonomy);
               cardData.push(mcqCard);
             }
           } else if (elem.type == "cloze") {
-            const clozeCard = new ParseClozeCard().parse(elem);
+            const clozeCard = new ParseClozeCard().parse({
+              card_content: elem.card_content as RawClozeCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (clozeCard && clozeCard != null) {
               clozeCard.heading = this._getCardReference(
                 clozeCard,
@@ -38,7 +88,12 @@ export class ParseCardResponse {
               cardData.push(clozeCard);
             }
           } else if (elem.type == "match") {
-            const matchCard = new ParseMatchCard().parse(elem);
+            const matchCard = new ParseMatchCard().parse({
+              card_content: elem.card_content as RawMatchCardResponseType,
+              type: elem.type,
+              concepts_facts: managedCardConcepts,
+              bloom_level: elem.bloom_level,
+            });
             if (matchCard && matchCard != null) {
               matchCard.heading = this._getCardReference(
                 matchCard,
@@ -60,41 +115,27 @@ export class ParseCardResponse {
       return {
         status_code: cardData.length == 0 ? 400 : 200,
         metadata: usage_data,
-        type: type,
-        missing_concepts: [],
-        missing_facts: [],
+        type: "card_gen",
+
         cards_data: cardData,
       };
     } catch (e: any) {
-      new ErrorLogger({
-        type: "card_parsing",
-        data: e.message,
-        response: generatedData,
-      }).log();
       return {
         status_code: 500,
         metadata: usage_data,
-        type: generatedData.type,
+        type: "card_gen",
       };
     }
   }
 
   _getCardReference(generatedCardData: any, sourceTaxonomy: any) {
-    const cardConcepts = generatedCardData.concepts ?? [];
-    const cardFacts = generatedCardData.facts ?? [];
-    const combinedCardFactsAndConcepts = [...cardConcepts, ...cardFacts];
-
-    const sourceConcepts = sourceTaxonomy.concepts ?? [];
-
-    const mappedSourceConcepts = sourceConcepts.map((elem: any) => {
-      return {
-        text: elem.concept_text,
-        reference: elem.reference,
-      };
-    });
+    const cardConcepts = (generatedCardData.concepts ?? []).map(
+      (e: any) => e.text
+    );
 
-    const firstMatchedConcept = sourceConcepts.find((elem: any) =>
-      combinedCardFactsAndConcepts.includes(elem.text)
+    const sourceConceptsFacts = sourceTaxonomy.concepts_facts ?? [];
+    const firstMatchedConcept = sourceConceptsFacts.find((elem: any) =>
+      cardConcepts.includes(elem.text)
     );
 
     if (firstMatchedConcept) {
@@ -103,4 +144,26 @@ export class ParseCardResponse {
       return "";
     }
   }
+
+  _mapIdToConcepts(
+    cardConcepts: any[],
+    sourceConceptsFacts: any[]
+  ): MongoConceptFactCards[] {
+    // const cardData = cardC
+    const managedCardConcepts = [];
+    for (const cardConcept of cardConcepts) {
+      const matchedConcept = sourceConceptsFacts.find(
+        (e: any) =>
+          e.text.toLowerCase().trim() == cardConcept.text.toLowerCase().trim()
+      );
+      if (matchedConcept) {
+        managedCardConcepts.push({
+          id: matchedConcept.id,
+          text: matchedConcept.text,
+          type: matchedConcept.type,
+        });
+      }
+    }
+    return managedCardConcepts;
+  }
 }

package/src/parse/parse_source_content.ts

@@ -1,3 +1,5 @@
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+
 export class ParseSourceContent {
   public content: any;
   /// Format of Content
@@ -25,7 +27,13 @@ export class ParseSourceContent {
     this.content = sourceContent;
   }
 
-  parseData() {
+  parseData(): {
+    type: string;
+    title: string;
+    content: any[];
+    headings: string[];
+    taxonomy: SourceTaxonomy;
+  } {
     let sourceType = this.content.type;
     let afterSanitized;
     if (sourceType == "video") {

package/src/services/open_ai_service.ts

@@ -3,7 +3,10 @@ import {
   parseOpenAiFailureResponse,
   parseOpenAiSuccessResponse,
 } from "../utils/parse_openai_response";
-import { openAiEndPoint } from "../constants/api_constants";
+import {
+  openAiEmbeddingEndPoint,
+  openAiEndPoint,
+} from "../constants/api_constants";
 
 export class OpenAiService {
   public api_key: string;
@@ -53,4 +56,33 @@ export class OpenAiService {
       return parseOpenAiFailureResponse(err.response);
     }
   }
+
+  async sendEmbeddingRequest(texts: string[]) {
+    try {
+      const url = openAiEmbeddingEndPoint();
+      let response = await axios.post(
+        url,
+        {
+          model: "text-embedding-3-large",
+          input: texts,
+          dimensions: 512,
+        },
+        {
+          headers: {
+            Authorization: "Bearer " + this.api_key,
+            "Content-Type": ["application/json"],
+          },
+        }
+      );
+      return {
+        status_code: 200,
+        data: response.data,
+      };
+    } catch (err: any) {
+      return {
+        status_code: err.response.status ?? 500,
+        message: err.message,
+      };
+    }
+  }
 }

package/src/services/qdrant_service.ts

@@ -0,0 +1,10 @@
+import { QdrantClient } from "@qdrant/js-client-rest";
+import config from "../config";
+
+// or connect to Qdrant Cloud
+const qdrantClient = new QdrantClient({
+  url: config.qdrantUrl,
+  apiKey: config.qdrantApiKey,
+});
+
+export default qdrantClient;

package/src/types/base_param_type.ts

@@ -0,0 +1,16 @@
+import { SourceTaxonomy } from "./source_taxonomy_type";
+
+export type BaseParamType = {
+  prompt: {
+    typology: string;
+    card_generation: string;
+  };
+  content: {
+    title: string;
+    headings: string[];
+    content: any[];
+    fields: string[];
+    taxonomy?: SourceTaxonomy;
+    type: string;
+  };
+};

package/src/types/mongo_concept_fact_type.ts

@@ -0,0 +1,12 @@
+export type MongoConceptFactSource = {
+  text: string;
+  type: "concept" | "fact";
+  reference: string;
+  id: string;
+};
+
+export type MongoConceptFactCards = {
+  text: string;
+  type: "concept" | "fact";
+  id: string;
+};

package/src/types/parsed_card_type.ts

@@ -0,0 +1,39 @@
+export type ParsedCardType = {
+  type: {
+    category: string;
+    sub_type: string;
+  };
+  heading: string;
+  displayTitle: string;
+  content: any;
+  explanation?: string;
+  bloom_level?: number;
+};
+
+// {
+//     "type": {
+//       "category": "learning",
+//       "sub_type": "cloze"
+//     },
+//     "heading": "",
+//     "displayTitle": "Ink is a medium that consists of colorants and other {{components}}. ---- components, elements, substances",
+//     "content": {
+//       "question": "Ink is a medium that consists of colorants and other {{c0:components}}.",
+//       "options": [
+//         {
+//           "option": "components",
+//           "cloze": "c0"
+//         },
+//         {
+//           "option": "elements",
+//           "cloze": "null"
+//         },
+//         {
+//           "option": "substances",
+//           "cloze": "null"
+//         }
+//       ]
+//     },
+//     "explanation": "Ink contains various components that include colorants, solvents, and additives which determine its properties.",
+//     "bloom_level": 1
+//   },

package/src/types/raw_card_response_types/generated_card_response_type.ts

@@ -0,0 +1,61 @@
+import { CardTypeEnum } from "../../enums/card_type_enum";
+
+export type GeneratedCardResponseType = {
+  metadata: any;
+  generated_content: {
+    test_cards: RawTestCardResponseType[];
+  };
+  status_code: number;
+  usage_data: any;
+  generated_at: Date;
+};
+
+export type RawTestCardResponseType = {
+  type: string;
+  // type:
+  //   | CardTypeEnum.FLASH
+  //   | CardTypeEnum.MATCH
+  //   | CardTypeEnum.MCQ
+  //   | CardTypeEnum.CLOZE;
+  card_content:
+    | RawFlashCardResponseType
+    | RawMatchCardResponseType
+    | RawClozeCardResponseType
+    | RawMcqCardResponseType;
+  concepts: string[];
+  facts: string[];
+  bloom_level: number;
+};
+
+export type RawFlashCardResponseType = {
+  front: string;
+  back: string;
+  explanation?: string;
+};
+
+export type RawMatchCardResponseType = {
+  left_item: string;
+  right_item: string;
+}[];
+
+export type RawClozeCardResponseType = {
+  correct_options: string[];
+  incorrect_options: string[];
+  prompt: string;
+  explanation?: string;
+};
+
+export type RawMcqCardResponseType = {
+  prompt: string;
+  explanation?: string;
+  choices: {
+    choice: string;
+    is_correct: boolean;
+  }[];
+};
+
+//
+// flash : 1
+// cloze 0,
+// match 2
+// mcq 3

package/src/types/source_taxonomy_type.ts

@@ -0,0 +1,17 @@
+export type SourceTaxonomy = {
+  concepts_facts: {
+    text: string;
+    type: string;
+    /// will only be present temporarily and wont be stored in database
+    embedding?: number[];
+    reference: string;
+  }[];
+  fields: string[];
+
+  /// will only be present temporarily and wont be stored in database
+  metadata?: any[];
+  generate_cards: {
+    state: boolean;
+    reason: string;
+  };
+};

package/src/typology_gen/generate_typology.ts

@@ -1,5 +1,5 @@
 import { OpenAiService } from "../services/open_ai_service";
-import { returnTypologyData } from "../parse/response_format_typology";
+
 import { ErrorLogger } from "../logger";
 
 export class GenerateTypology {
@@ -57,26 +57,27 @@ export class GenerateTypology {
 
   parseTypologyOnSuccess(responseData: any) {
     responseData.metadata.status = "completed";
-    const facts = responseData.generated_content.facts.map((fact: any) => {
+    const generatedContent = responseData.generated_content;
+    const concepts = generatedContent.concepts.map((e: any) => {
       return {
-        concept_text: fact.fact_text,
-        reference: fact.reference,
-        type: "fact",
+        text: e.concept_text,
+        type: "concept",
+        reference: e.reference,
       };
     });
-    const concepts = responseData.generated_content.concepts.map((concept: any) => {
+    const facts = generatedContent.facts.map((e: any) => {
       return {
-        concept_text: concept.concept_text,
-        reference: concept.reference,
-        type: "concept",
+        text: e.fact_text,
+        type: "fact",
+        reference: e.reference,
       };
     });
-    const generatedContent = responseData.generated_content;
+
     return {
       status_code: 200,
-      metadata: responseData.metadata,
+      metadata: [responseData.metadata],
       field: this.parseFields(generatedContent.field),
-      concepts: [...concepts, ...facts],
+      concepts_facts: [...concepts, ...facts],
       generate_cards: generatedContent.generate_cards,
       summary_cards: generatedContent.summary_cards,
       // type: responseData.type,

package/tsconfig.json

@@ -5,8 +5,13 @@
     "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
     "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
     "strict": true,
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": false,
     "rootDir": "./src",
-    "outDir": "./dist" /* Enable all strict type-checking options. */
+    "outDir": "./dist", /* Enable all strict type-checking options. */
+    "sourceMap": true
   },
-  "exclude": ["./src/*/*.d.ts"]
-}
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "**/*.test.ts"]
+}  

package/.env.example

@@ -1 +0,0 @@
-OPEN_AI_KEY= 'open ai key'

package/prompts.json

@@ -1,23 +0,0 @@
-[
-    {
-        "generation_for":"card",
-        "type": "role",
-        "prompt": "As a dedicated assistant at a learning company, your role is to create test cards based on the provided content. The purpose of these cards is to help learners master and understand the concepts and facts presented to you."
-    },
-    {
-        "generation_for":"card",
-        "type": "inputs",
-        "prompt": "You will be provided with the following: \n 1. Title of the source \n 2. The content \n 3. The field of knowledge it belongs to \n 4. Key concepts in the source \n 5. Important facts in the source"
-    },
-    {
-        "generation_for":"card",
-        "type" :"steps",
-        "prompt": "**Follow these steps:** \n 1. Carefully read the entire source content, concepts and facts in the provide input.\n  2. Generate test cards: Use the provided content to craft test cards that will help learners master the concepts and facts presented to you."
-    },
-    {
-        "generation_for": "card",
-        "type": "schema",
-        "prompt": "**Format your response in the following JSON format:** json { \"test_cards\": [ { \"type\": \"{card_type}\", \"card_content\": \"{content}\", \"concepts\": [\"concept1\", \"concept2\", \"...\"], \"facts\": [\"fact1\", \"fact2\", \"...\"] }, {... as many cards as possible} ] }"
-
-    }
-]

package/readme.md

@@ -1,35 +0,0 @@
-This is a NPM package that handles the all the ai generation for _source on OnlyEver.
-The Generation includes, Knowledge Taxonomy , Summary and Test Cards data for the _source.
-
-The class [OnlyEverGenerator] is the Entry point for the generation module it has two required fields, 
-[api_key] which is open ai token 
-[model] open ai model : 'gpt-3.5-turbo-1106'
-[content] source content : Array<BaseBlocks>
-
-The [OnlyEverGenerator] currently exposes a main method, [generate] which handles all types of generation and takes a Param 
-```
-generate(generate_typology = false,generate_card = false) method takes two args, generate_typology and generate_cards, which are both boolean.
-For Eg: 
-let oeGenerator = new OnlyEverGenerator(
-    api_key,
-    model,
-    [Block1,Block2,Block3], // oe content model
-);
- 
-
-let responsesArray = await oeGenerator.generate(true,true);
-
-
-Exposed a new method ```gapFill``` from OnlyEverGenerator
-
-
-  async gapFill(factsMaps: any, aiCards: Array<any>) {}
-    /// factsmap 
-    /// {
-    /// remaining_facts: [],
-     /// remaining_concepts: [],
-    //}
-
-    /// aicards is array of ai cards
-
-```