only_ever_generator 1.0.3 → 1.0.5

package/readme.md

@@ -8,25 +8,28 @@ The class [OnlyEverGenerator] is the Entry point for the generation module it ha
 
 The [OnlyEverGenerator] currently exposes a main method, [generate] which handles all types of generation and takes a Param 
 ```
-GenerateArgs : 
+generate(generate_typology = false,generate_card = false) method takes two args, generate_typology and generate_cards, which are both boolean.
 For Eg: 
 let oeGenerator = new OnlyEverGenerator(
     api_key,
     model,
     [Block1,Block2,Block3], // oe content model
 );
-    let args = new GenerateArgs(
-       true,
-       true,
-       false,
-        {
-         typology_prompt: typologyPrompt, 
-         card_gen_prompt: cardPrompt,
-         summary_prompt: summary prompt,
-       }
-    )
-
-let responsesArray = await oeGenerator.generate(args);
+ 
 
+let responsesArray = await oeGenerator.generate(true,true);
+
+
+Exposed a new method ```gapFill``` from OnlyEverGenerator
+
+
+  async gapFill(factsMaps: any, aiCards: Array<any>) {}
+    /// factsmap 
+    /// {
+    /// remaining_facts: [],
+     /// remaining_concepts: [],
+    //}
+
+    /// aicards is array of ai cards
 
 ```

package/src/bootstrap/app.ts

@@ -1,107 +1,167 @@
-// import { ParseSourceContent } from "../class/parse/parse_source_content";
-// import { OpenAiService } from "../class/services/open_ai_service";
-
-
-
-// export class OnlyEverGenerator{
-//     public api_key: string = '';
-//     public openAiService: OpenAiService | undefined;
-//     parsedContent: String = '';
-//     constructor(apiKey:string, content: Array<any>){
-//         this.api_key = apiKey;
-//         this.openAiService = new OpenAiService(apiKey);
-//         this.parsedContent = new ParseSourceContent(content).parse();
-//     };
-
-//     typologyResponse = {};
-//     cardgenResponse = {};
-//     summarizeResponse = {};
-
-    
-    
-
-//     async generate( 
-//         args: GenerateArgs
-//     ): Promise<Array<any>> {
-//         const responseToReturn = [];
-//         const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
-//         for(let elem of whatNeedsToBeGenerated)
-//         if(elem == 'generate_tyopology'){
-//             this.typologyResponse = await this.generateTypology(args.prompts.typology_prompt ?? '', this.parsedContent);
-//             responseToReturn.push(this.typologyResponse);
-//         }else if(elem == 'generate_card'){
-
-//             this.cardgenResponse =  await this.generateCard(args.prompts.card_gen_prompt ?? '', this.parsedContent +  JSON.stringify(this.typologyResponse));
-//             responseToReturn.push(this.cardgenResponse);
-//         }else if(elem == 'generate_summary'){
-//             this.summarizeResponse = await this.generateSummary(args.prompts.summary_prompt ?? '', this.parsedContent);
-//             responseToReturn.push(this.summarizeResponse);
-//         }
-
-//         return responseToReturn;
-        
-//     }
-    
-//     returnParsedContent(){
-//         return this.parsedContent;
-//     }
-
-
-//     async generateCard(prompt: String, content: String){
-//         let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-//         response['type'] = 'card_gen';
-//         return response;
-//     }
-
-//     async generateTypology(prompt:String,content:String){
-//         let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-//         response['type'] = 'typology';
-//         return response;
-//     }
-
-//     async generateSummary(prompt:String,content:String){
-//         let response = await this.openAiService?.sendRequest(prompt,this.parsedContent);
-//         response['type']= 'summary';
-//         return response;
-//     }
-    
-// }
-
-// export class GenerateArgs{
-//     public generate_card : boolean = false;
-//     public generate_typology: boolean = false;
-//     public generate_summary: boolean = false;
-//     public prompts = {
-//         typology_prompt: '',
-//         card_gen_prompt: '',
-//         summary_prompt: ''
-//     }
-
-//     constructor(generate_card: boolean,generate_typology: boolean, generate_summary: boolean, prompts = {
-//         typology_prompt: '',
-//         card_gen_prompt: '',
-//         summary_prompt: ''
-//     }){
-//         this.generate_card = generate_card;
-//         this.generate_typology = generate_typology;
-//         this.generate_summary = generate_summary;
-//         this.prompts = prompts
-//     }
-
-//     getWhatNeedsToBeGenerated(){
-//         let returnData = [];
-//         if(this.generate_typology == true){
-//             returnData.push('generate_tyopology')
-//         }
-//         if(this.generate_summary == true){
-//             returnData.push('generate_summary')
-//         }
-//         if(this.generate_card == true){
-//             returnData.push('generate_card');
-//         }
-//         return returnData;
-        
-        
-//     }
-
-// }
+import { GenerateCards } from "../card_gen/generate_cards";
+import { ParseSourceContent } from "../parse/parse_source_content";
+import { OpenAiService } from "../services/open_ai_service";
+import { GenerateTypology } from "../typology_gen/generate_typology";
+import { GenerateArgs } from "../utils/generate_args";
+import { GenerateEmbeddings } from "../embedding_generation/generate_embeddings";
+import { LocalConsolidation } from "../embedding_generation/consolidation/local_consolidation";
+import { GlobalConsolidation } from "../embedding_generation/consolidation/global_consolidation";
+import { WriteConsolidatedData } from "../embedding_generation/consolidation/write_consolidated_data";
+
+export class OnlyEverGenerator {
+  public api_key: string = "";
+  public openAiService: OpenAiService;
+
+  /// these fields will be populated inside the constructor
+  parsedContent: any = {};
+  promptForTypology: string = "";
+  promptForCardGen: string = "";
+  expectedFields: Array<string>;
+
+  typologyResponse: any = undefined;
+  cardgenResponse: any = undefined;
+  summarizeResponse = {};
+  gapFillResponse: any = {};
+
+  constructor(apiKey: string, model: string, generationContent: any) {
+    this.api_key = apiKey;
+    this.openAiService = new OpenAiService(
+      apiKey,
+      model ?? "gpt-3.5-turbo-1106"
+    );
+    const parsedData = new ParseSourceContent(
+      generationContent.content
+    ).parseData();
+    (this.parsedContent = {
+      title: parsedData.title,
+      headings: parsedData.headings,
+      content: parsedData.content,
+      taxonomy: parsedData.taxonomy,
+    }),
+      (this.typologyResponse = generationContent.content.taxonomy);
+
+    this.expectedFields = generationContent.content.fields; //returnFields();
+    this.promptForTypology = generationContent.prompt.typology;
+    this.promptForCardGen = generationContent.prompt.card_generation;
+  }
+
+  async generate(
+    generate_typology: boolean = false,
+    generate_card: boolean = false
+  ): Promise<Array<any>> {
+    let args = new GenerateArgs(generate_card, generate_typology, false);
+    const responseToReturn = [];
+    const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
+    for (let elem of whatNeedsToBeGenerated)
+      if (elem == "generate_tyopology") {
+        this.typologyResponse = await this.generateTypology(
+          this.promptForTypology
+        );
+
+        const embeddings = await this.generateEmbeddings(this.typologyResponse);
+        this.typologyResponse.concepts_facts = embeddings.concepts_facts;
+        this.typologyResponse.metadata.push(embeddings.metadata);
+        responseToReturn.push(this.typologyResponse);
+      } else if (elem == "generate_card") {
+        if (this.shouldTheCardBeGeneratedAfterTypologyResponse()) {
+          this.parsedContent.taxonomy = {
+            concepts_facts: this.typologyResponse.concepts_facts,
+            generate_cards: this.typologyResponse.generate_cards,
+          };
+          this.cardgenResponse = await this.generateCard(
+            this.promptForCardGen,
+            JSON.stringify(this.typologyResponse),
+            false
+          );
+          responseToReturn.push(this.cardgenResponse);
+        }
+      }
+    return responseToReturn;
+  }
+
+  shouldTheCardBeGeneratedAfterTypologyResponse() {
+    if (this.typologyResponse) {
+      return this.typologyResponse?.generate_cards?.state == true;
+    } else {
+      return false;
+    }
+  }
+
+  async generateCard(
+    prompt: string,
+    additionalContent: string,
+    isGapFill: boolean,
+    n: number = 0
+  ) {
+    let generateCardsResp = await new GenerateCards(
+      this.openAiService
+    ).generateCards(
+      prompt ?? "",
+      JSON.stringify(this.parsedContent) + additionalContent,
+      isGapFill,
+      this.parsedContent.taxonomy,
+      n
+    );
+    return generateCardsResp;
+  }
+
+  async generateTypology(prompt: string) {
+    let response = await new GenerateTypology(
+      this.openAiService,
+      prompt,
+      JSON.stringify(this.parsedContent),
+      this.expectedFields
+    ).generate();
+    return response;
+  }
+
+  async generateEmbeddings(typologyResponse: any) {
+    const concepts_facts = typologyResponse.concepts_facts;
+    const texts = concepts_facts.map((e: any) => e.text);
+    const embeddings = await new GenerateEmbeddings(
+      this.openAiService
+    ).generateEmbeddings(concepts_facts);
+    return embeddings;
+  }
+
+  async localConsolidation(
+    concepts_facts: {
+      text: string;
+      type: string;
+      reference: string;
+      embedding: number[];
+    }[],
+    sourceId: string
+  ) {
+    const embeddings = new LocalConsolidation().consolidate(
+      concepts_facts,
+      sourceId
+    );
+    return embeddings;
+  }
+
+  async globalConsolidation(
+    concepts_facts: {
+      id: string;
+      text: string;
+      type: string;
+      reference: string;
+      embedding: number[];
+    }[],
+    sourceId: string,
+    threshold: number
+  ) {
+    const embeddings = await new GlobalConsolidation().consolidate(
+      concepts_facts,
+      sourceId,
+      threshold ?? 0.8
+    );
+    // const writeConsolidatedData =
+    //   new WriteConsolidatedData().writeConsolidatedData(
+    //     embeddings,
+    //     [],
+    //     sourceId
+    //   );
+    return embeddings;
+  }
+}

package/src/card_gen/generate_cards.ts

@@ -0,0 +1,58 @@
+import { ErrorLogger } from "../logger";
+import { ParseCardResponse } from "../parse/parse_card_response";
+import { OpenAiService } from "../services/open_ai_service";
+
+export class GenerateCards {
+  openAiService: OpenAiService;
+  constructor(openAiService: OpenAiService) {
+    this.openAiService = openAiService;
+  }
+
+  async generateCards(
+    prompt: string,
+    parsedContent: string,
+    isGapFill: boolean,
+    taxonomy: any,
+    n: number = 0
+  ) {
+    try {
+      let response = await this.openAiService?.sendRequest(
+        prompt,
+        parsedContent
+      );
+      var updatedNumber = n + 1;
+      // console.log("response to card generation ", response);
+      // response["request_type"] = ;
+      response.metadata = {
+        req_time: response.generated_at ?? new Date(),
+        req_type: {
+          type: "depth",
+          n: updatedNumber,
+          bloom_level: 1,
+        },
+        req_tokens: response.usage_data?.prompt_tokens,
+        res_tokens: response.usage_data?.completion_tokens,
+        prompt_tokens_details: response.usage_data?.prompt_tokens_details,
+        model: this.openAiService.model,
+      };
+      if (response.status_code == 200) {
+        response.metadata.status = "completed";
+        let parseCard = new ParseCardResponse().parse(
+          response,
+          isGapFill,
+          taxonomy
+        );
+        return parseCard;
+      } else {
+        response.metadata.status = "failed";
+        response.metadata.err_message = response.message;
+        return response;
+      }
+    } catch (e: any) {
+      new ErrorLogger({
+        type: "card_generation",
+        data: e.message,
+      }).log();
+    }
+  }
+}

package/src/config.ts

@@ -1,7 +1,9 @@
 import "dotenv/config";
 
 const config = {
-    openAIKey  : process.env.OPEN_AI_KEY as string
-}
+  openAIKey: process.env.OPEN_AI_KEY as string,
+  qdrantApiKey: process.env.QDRANT_API_KEY,
+  qdrantUrl: process.env.QDRANT_URL,
+};
 
-export default config;
+export default config;

package/src/constants/api_constants.ts

@@ -0,0 +1,7 @@
+export function openAiEndPoint() {
+  return "https://api.openai.com/v1/chat/completions";
+}
+
+export function openAiEmbeddingEndPoint() {
+  return "https://api.openai.com/v1/embeddings";
+}