only_ever_generator 1.0.7 → 1.0.9

package/src/bootstrap/app.ts

@@ -7,28 +7,67 @@ import { GenerateEmbeddings } from "../embedding_generation/generate_embeddings"
 import { LocalConsolidation } from "../embedding_generation/consolidation/local_consolidation";
 import { GlobalConsolidation } from "../embedding_generation/consolidation/global_consolidation";
 import { WriteConsolidatedData } from "../embedding_generation/consolidation/write_consolidated_data";
+import { BaseParamType } from "../types/base_param_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+import { MongoConceptFactSource } from "../types/mongo_concept_fact_type";
+import { OpenAIHelper } from "../helper/openai_helper";
+import { GenerateConceptFacts } from "../typology_gen/generate_concept_facts";
 
 export class OnlyEverGenerator {
   public api_key: string = "";
   public openAiService: OpenAiService;
 
   /// these fields will be populated inside the constructor
-  parsedContent: any = {};
+  parsedContent?: {
+    title: string;
+    headings: string[];
+    content: any[];
+    taxonomy: SourceTaxonomy;
+    source_id: string;
+    type: string;
+  } = undefined;
   promptForTypology: string = "";
   promptForCardGen: string = "";
-  expectedFields: Array<string>;
-
-  typologyResponse: any = undefined;
+  promptForSummary: {
+    promptId: string;
+    version: string;
+  } = {
+    promptId: "",
+    version: "",
+  };
+  promptForConceptFacts: {
+    promptId: string;
+    version: string;
+  } = {
+    promptId: "",
+    version: "",
+  };
+  expectedFields: Array<string> = [];
+  typologyResponse?: any = undefined;
   cardgenResponse: any = undefined;
   summarizeResponse = {};
-  gapFillResponse: any = {};
 
-  constructor(apiKey: string, model: string, generationContent: any) {
+  constructor(
+    apiKey: string,
+    model: string,
+    generationContent: BaseParamType,
+    promptForSummary: {
+      promptId: string;
+      version: string;
+    },
+    promptForConceptFacts: {
+      promptId: string;
+      version: string;
+    }
+  ) {
+    this.promptForSummary = promptForSummary;
+    this.promptForConceptFacts = promptForConceptFacts;
     this.api_key = apiKey;
     this.openAiService = new OpenAiService(
       apiKey,
       model ?? "gpt-3.5-turbo-1106"
     );
+
     const parsedData = new ParseSourceContent(
       generationContent.content
     ).parseData();
@@ -37,6 +76,8 @@ export class OnlyEverGenerator {
       headings: parsedData.headings,
       content: parsedData.content,
       taxonomy: parsedData.taxonomy,
+      source_id: parsedData.source_id,
+      type: parsedData.type,
     }),
       (this.typologyResponse = generationContent.content.taxonomy);
 
@@ -54,26 +95,69 @@ export class OnlyEverGenerator {
     const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
     for (let elem of whatNeedsToBeGenerated)
       if (elem == "generate_tyopology") {
-        this.typologyResponse = await this.generateTypology(
-          this.promptForTypology
-        );
+        this.typologyResponse = await this.generateTypology();
+        console.log("Typology generated");
+        console.log(this.typologyResponse);
 
-        const embeddings = await this.generateEmbeddings(this.typologyResponse);
-        this.typologyResponse.concepts_facts = embeddings.concepts_facts;
-        this.typologyResponse.metadata.push(embeddings.metadata);
-        responseToReturn.push(this.typologyResponse);
+        if (this.typologyResponse?.status_code == 200) {
+          console.log("Concept facts generating");
+          const response = await this.generateConceptFacts();
+          this.typologyResponse.concepts_facts = response.concepts_facts;
+          console.log("Concept facts generated");
+          console.log(this.typologyResponse);
+          const embeddings = await this.generateEmbeddings(
+            this.typologyResponse
+          );
+          console.log("Embedding generated");
+          const localConsolidation = await this.localConsolidation(
+            embeddings.concepts_facts,
+            this.parsedContent!.title
+          );
+          console.log("Local consolidation done");
+          this.typologyResponse!.concepts_facts =
+            localConsolidation.sourceTaxonomyOps;
+          const globalConsolidation = await this.globalConsolidation(
+            localConsolidation.sourceTaxonomyOps,
+            this.parsedContent!.source_id,
+            0.8
+          );
+          console.log("Global consolidation done");
+          this.typologyResponse!.concepts_facts =
+            globalConsolidation.source_taxonomy;
+          this.typologyResponse!.metadata?.push(embeddings.metadata);
+          responseToReturn.push({
+            type: "typology",
+            data: this.typologyResponse,
+            qdrantOps: {
+              update: globalConsolidation.global_updates,
+              inserts: globalConsolidation.global_inserts,
+            },
+          });
+        } else {
+          responseToReturn.push({
+            type: "typology",
+            data: this.typologyResponse,
+          });
+        }
       } else if (elem == "generate_card") {
         if (this.shouldTheCardBeGeneratedAfterTypologyResponse()) {
-          this.parsedContent.taxonomy = {
-            concepts_facts: this.typologyResponse.concepts_facts,
-            generate_cards: this.typologyResponse.generate_cards,
+          this.parsedContent!.taxonomy = {
+            concepts_facts: this.typologyResponse?.concepts_facts ?? [],
+            fields: this.expectedFields,
+            generate_cards: this.typologyResponse?.generate_cards ?? {
+              state: false,
+              reason: "",
+            },
           };
           this.cardgenResponse = await this.generateCard(
             this.promptForCardGen,
             JSON.stringify(this.typologyResponse),
             false
           );
-          responseToReturn.push(this.cardgenResponse);
+          responseToReturn.push({
+            type: "card_gen",
+            data: this.cardgenResponse,
+          });
         }
       }
     return responseToReturn;
@@ -99,25 +183,46 @@ export class OnlyEverGenerator {
       prompt ?? "",
       JSON.stringify(this.parsedContent) + additionalContent,
       isGapFill,
-      this.parsedContent.taxonomy,
+      this.parsedContent!.taxonomy,
       n
     );
     return generateCardsResp;
   }
 
-  async generateTypology(prompt: string) {
+  async generateTypology() {
+    const content = {
+      title: this.parsedContent?.title,
+      headings: this.parsedContent?.headings,
+      content: this.parsedContent?.content,
+    };
     let response = await new GenerateTypology(
       this.openAiService,
-      prompt,
-      JSON.stringify(this.parsedContent),
-      this.expectedFields
+      this.parsedContent?.type ?? "wiki",
+      JSON.stringify(content),
+      this.expectedFields,
+      this.promptForSummary
+    ).generate();
+    return response;
+  }
+
+  async generateConceptFacts() {
+    const content = {
+      title: this.parsedContent?.title,
+      headings: this.parsedContent?.headings,
+      content: this.parsedContent?.content,
+    };
+    let response = await new GenerateConceptFacts(
+      this.openAiService,
+      JSON.stringify(content),
+      this.parsedContent?.type ?? "wiki",
+      this.promptForConceptFacts
     ).generate();
     return response;
   }
 
   async generateEmbeddings(typologyResponse: any) {
     const concepts_facts = typologyResponse.concepts_facts;
-    const texts = concepts_facts.map((e: any) => e.text);
+    const texts = concepts_facts.map((e: MongoConceptFactSource) => e.text);
     const embeddings = await new GenerateEmbeddings(
       this.openAiService
     ).generateEmbeddings(concepts_facts);
@@ -156,12 +261,7 @@ export class OnlyEverGenerator {
       sourceId,
       threshold ?? 0.8
     );
-    // const writeConsolidatedData =
-    //   new WriteConsolidatedData().writeConsolidatedData(
-    //     embeddings,
-    //     [],
-    //     sourceId
-    //   );
+
     return embeddings;
   }
 }

package/src/card_gen/generate_cards.ts

@@ -1,6 +1,8 @@
 import { ErrorLogger } from "../logger";
 import { ParseCardResponse } from "../parse/parse_card_response";
 import { OpenAiService } from "../services/open_ai_service";
+import { GeneratedCardResponseType } from "../types/raw_card_response_types/generated_card_response_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
 
 export class GenerateCards {
   openAiService: OpenAiService;
@@ -12,7 +14,7 @@ export class GenerateCards {
     prompt: string,
     parsedContent: string,
     isGapFill: boolean,
-    taxonomy: any,
+    taxonomy: SourceTaxonomy,
     n: number = 0
   ) {
     try {
@@ -21,8 +23,6 @@ export class GenerateCards {
         parsedContent
       );
       var updatedNumber = n + 1;
-      // console.log("response to card generation ", response);
-      // response["request_type"] = ;
       response.metadata = {
         req_time: response.generated_at ?? new Date(),
         req_type: {
@@ -35,12 +35,34 @@ export class GenerateCards {
         prompt_tokens_details: response.usage_data?.prompt_tokens_details,
         model: this.openAiService.model,
       };
+
       if (response.status_code == 200) {
         response.metadata.status = "completed";
+        /// map response to generated_content
+        const generated_data: GeneratedCardResponseType = {
+          metadata: response.metadata,
+          generated_content: {
+            test_cards: (response.generated_content.test_cards ?? []).map(
+              (card: any) => {
+                return {
+                  type: card.type,
+                  card_content: card.card_content,
+                  concepts: card.concepts ?? [],
+                  facts: card.facts ?? [],
+                  bloom_level: card.bloom_level ?? 1,
+                };
+              }
+            ),
+          },
+          usage_data: response.usage_data,
+          generated_at: response.generated_at,
+          status_code: response.status_code,
+        };
         let parseCard = new ParseCardResponse().parse(
-          response,
+          generated_data,
           isGapFill,
-          taxonomy
+          taxonomy,
+          1
         );
         return parseCard;
       } else {

package/src/constants/prompts/card_gen_prompt.ts

@@ -160,5 +160,5 @@ Once you are done generating the test cards, review the full list of concepts an
 Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.
 `;
 export function returnCardGenPrompt() {
-    return promptString;
+  return promptString;
 }

package/src/embedding_generation/consolidation/global_consolidation.ts

@@ -27,8 +27,8 @@ export class GlobalConsolidation {
 
     const globalUpdatesOps: {
       id: string;
-      vector: number[];
       sourceIdToAdd: string;
+      vector: number[];
       currentPayload: {
         _sources: string[];
         text: string;
@@ -73,8 +73,8 @@ export class GlobalConsolidation {
         });
         globalUpdatesOps.push({
           id: consolidatedId.toString(),
-          vector: originalConcept.embedding,
           sourceIdToAdd: sourceId,
+          vector: originalConcept.embedding,
           currentPayload: {
             // _sources: currentPayload['_sources'] ?? [],
             _sources: (currentPayload?._sources ?? []) as string[],

package/src/enums/card_type_enum.ts

@@ -0,0 +1,6 @@
+export enum CardTypeEnum {
+  FLASH = "flash",
+  CLOZE = "cloze",
+  MATCH = "match",
+  MCQ = "mcq",
+}

package/src/helper/openai_helper.ts

@@ -0,0 +1,31 @@
+import OpenAI from "openai";
+
+export class OpenAIHelper {
+  public openAI: OpenAI;
+  public model: string;
+  public promptId: string;
+  public openAIKey: string;
+  public version: string;
+  constructor(
+    openAIKey: string,
+    model: string,
+    promptId: string,
+    version: string
+  ) {
+    this.openAI = getOpenAI(openAIKey);
+    this.model = model;
+    this.promptId = promptId;
+    this.openAIKey = openAIKey;
+    this.version = version;
+  }
+}
+
+const getOpenAI = (openAIKey: string) => {
+  const openai = new OpenAI({
+    apiKey: openAIKey,
+  });
+
+  return openai;
+};
+
+export { getOpenAI, OpenAI };