only_ever_generator 0.2.6 → 0.2.8

package/dist/parse/parse_source_content.js

@@ -7,18 +7,32 @@ class ParseSourceContent {
         this.content = sourceContent;
     }
     parseData() {
-        if (this.content.type == 'source') {
-            let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(this.content.content);
-            let afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
-            return JSON.stringify([
-                { 'title': this.content.title },
-                { 'headings': this.content.headings },
-                { 'content': afterSanitized }
-            ]);
+        // if(this.content.type == 'source') {
+        let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(this.content.content);
+        let afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
+        let taxonomy;
+        if (this.content.taxonomy) {
+            taxonomy = this.content.taxonomy;
         }
         else {
-            return '';
+            taxonomy = null;
         }
+        return {
+            type: this.content.type,
+            title: this.content.title,
+            content: afterSanitized,
+            headings: this.content.headings,
+            taxonomy: this.content.taxonomy,
+        };
+        // } else if(this.content.type == 'cards'){
+        //     return {
+        //         type :'card',
+        //         title: this.content.title,
+        //         content: afterSanitized,
+        //         headings: this.content.headings
+        //         taxonomy: this.content.taxonomy,
+        //     };
+        // }
         //  return JSON.stringify(afterSanitized);    
     }
     removeSectionsByTitle(data) {

package/dist/typology_gen/generate_typology.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GenerateTypology = void 0;
+const logger_1 = require("../logger");
 class GenerateTypology {
     constructor(openAiService, prompt, content, expected_fields) {
         this.prompt = '';
@@ -22,19 +23,27 @@ class GenerateTypology {
     generate() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
-            const response = yield ((_a = this.openAiService) === null || _a === void 0 ? void 0 : _a.sendRequest(this.prompt, this.content));
-            response['type'] = 'typology';
-            response.metadata = {
-                "req_time": response.generated_at,
-                "req_type": response.type,
-                "req_tokens": (_b = response.usage_data) === null || _b === void 0 ? void 0 : _b.prompt_tokens,
-                "res_tokens": (_c = response.usage_data) === null || _c === void 0 ? void 0 : _c.completion_tokens,
-            };
-            if (response.status_code == 200) {
-                return this.parseTypologyOnSuccess(response);
+            try {
+                const response = yield ((_a = this.openAiService) === null || _a === void 0 ? void 0 : _a.sendRequest(this.prompt, this.content));
+                response['type'] = 'typology';
+                response.metadata = {
+                    "req_time": response.generated_at,
+                    "req_type": response.type,
+                    "req_tokens": (_b = response.usage_data) === null || _b === void 0 ? void 0 : _b.prompt_tokens,
+                    "res_tokens": (_c = response.usage_data) === null || _c === void 0 ? void 0 : _c.completion_tokens,
+                };
+                if (response.status_code == 200) {
+                    return this.parseTypologyOnSuccess(response);
+                }
+                else {
+                    return response;
+                }
             }
-            else {
-                return response;
+            catch (e) {
+                yield new logger_1.ErrorLogger({
+                    "type": 'typology_parsing',
+                    "data": e.message
+                }).log();
             }
         });
     }

package/dist/utils/generate_args.js

@@ -2,23 +2,13 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GenerateArgs = void 0;
 class GenerateArgs {
-    constructor(generate_card, generate_typology, generate_summary, prompts = {
-        typology_prompt: '',
-        card_gen_prompt: '',
-        summary_prompt: ''
-    }) {
+    constructor(generate_card, generate_typology, generate_summary) {
         this.generate_card = false;
         this.generate_typology = false;
         this.generate_summary = false;
-        this.prompts = {
-            typology_prompt: '',
-            card_gen_prompt: '',
-            summary_prompt: ''
-        };
         this.generate_card = generate_card;
         this.generate_typology = generate_typology;
         this.generate_summary = generate_summary;
-        this.prompts = prompts;
     }
     getWhatNeedsToBeGenerated() {
         let returnData = [];

package/package.json

@@ -1,33 +1,33 @@
-{
-  "name": "only_ever_generator",
-  "version": "0.2.6",
-  "main": "dist/index.js",
-  "scripts": {
-    "start": "npm run build && nodemon dist/index.js",
-    "build": "tsc",
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "keywords": [],
-  "author": "shree",
-  "license": "ISC",
-  "devDependencies": {
-    "@types/express": "^4.17.21",
-    "@types/node": "^20.14.2",
-    "nodemon": "^3.1.3",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.4.5",
-    "dotenv": "^16.4.5",
-    "express": "^4.19.2"
-  },
-  "dependencies": {
-    "axios": "^1.7.2"
- 
-
-  },
-  "eslintConfig": {
-    "parser": "typescript-eslint-parser",
-    "plugins": [
-      "typescript"
-    ]
-  }
-}
+{
+  "name": "only_ever_generator",
+  "version": "0.2.8",
+  "main": "dist/index.js",
+  "scripts": {
+    "start": "npm run build && nodemon dist/index.js",
+    "build": "tsc",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "shree",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.14.2",
+    "nodemon": "^3.1.3",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2"
+  },
+  "dependencies": {
+    "axios": "^1.7.2"
+ 
+
+  },
+  "eslintConfig": {
+    "parser": "typescript-eslint-parser",
+    "plugins": [
+      "typescript"
+    ]
+  }
+}

package/readme.md

@@ -1,23 +1,23 @@
-This is a NPM package that handles the all the ai generation for _source on OnlyEver.
-The Generation includes, Knowledge Taxonomy , Summary and Test Cards data for the _source.
-
-The class [OnlyEverGenerator] is the Entry point for the generation module it has two required fields, 
-[api_key] which is open ai token 
-[model] open ai model : 'gpt-3.5-turbo-1106'
-[content] source content : Array<BaseBlocks>
-
-The [OnlyEverGenerator] currently exposes a main method, [generate] which handles all types of generation and takes a Param 
-```
-generate(generate_typology = false,generate_card = false) method takes two args, generate_typology and generate_cards, which are both boolean.
-For Eg: 
-let oeGenerator = new OnlyEverGenerator(
-    api_key,
-    model,
-    [Block1,Block2,Block3], // oe content model
-);
- 
-
-let responsesArray = await oeGenerator.generate(true,true);
-
-
-```
+This is a NPM package that handles the all the ai generation for _source on OnlyEver.
+The Generation includes, Knowledge Taxonomy , Summary and Test Cards data for the _source.
+
+The class [OnlyEverGenerator] is the Entry point for the generation module it has two required fields, 
+[api_key] which is open ai token 
+[model] open ai model : 'gpt-3.5-turbo-1106'
+[content] source content : Array<BaseBlocks>
+
+The [OnlyEverGenerator] currently exposes a main method, [generate] which handles all types of generation and takes a Param 
+```
+generate(generate_typology = false,generate_card = false) method takes two args, generate_typology and generate_cards, which are both boolean.
+For Eg: 
+let oeGenerator = new OnlyEverGenerator(
+    api_key,
+    model,
+    [Block1,Block2,Block3], // oe content model
+);
+ 
+
+let responsesArray = await oeGenerator.generate(true,true);
+
+
+```

package/src/bootstrap/app.ts

@@ -1,141 +1,150 @@
-import { GenerateCards } from "../card_gen/generate_cards";
-import { ParseSourceContent } from "../parse/parse_source_content";
-import { OpenAiService } from "../services/open_ai_service";
-import { returnCardGenPrompt } from "../constants/prompts/card_gen_prompt";
-import { returnTypologyPrompt } from "../constants/prompts/typology_prompt";
-import { GenerateTypology } from "../typology_gen/generate_typology";
-import { GenerateArgs } from "../utils/generate_args";
-import { returnFields } from "../constants/source_data";
-import { returnTypologyData } from "../parse/response_format_typology";
-import { gapFilling } from "../gap_fill/calculate_gap_fill";
-
-
-/// OnlyEverGenerator
-
-export class OnlyEverGenerator {
-  public api_key: string = "";
-  public openAiService: OpenAiService;
-  parsedContent: string = "";
-  promptForTypology: string = "";
-  promptForCardGen: string = "";
-  expectedFields: Array<string>;
-  constructor(
-    apiKey: string,
-    model: string,
-    generationContent : any
-    // prompt: any,
-    // content: any,
-    // expected_fields: Array<string>
-  ) {
-    this.api_key = apiKey;
-    this.openAiService = new OpenAiService(
-      apiKey,
-      model ?? "gpt-3.5-turbo-1106"
-    );
-    
-    this.parsedContent = new ParseSourceContent(generationContent.content).parseData();
-    this.expectedFields =  generationContent.content.fields; //returnFields();
-    this.promptForTypology = returnTypologyPrompt(generationContent.prompt.typology);
-    this.promptForCardGen = returnCardGenPrompt(generationContent.prompt.card_generation);
-  }
-
-  typologyResponse: any = {};
-  cardgenResponse: any = {};
-  summarizeResponse = {};
-  gapFillResponse: any = {};
-
- 
-  async generate(
-    generate_typology: boolean = false,
-    generate_card: boolean = false
-  ): Promise<Array<any>> {
-    //let typologyPrompt = returnTypologyPrompt();
-    let typologyPrompt = this.promptForTypology;
-
-   // let cardPrompt = returnCardGenPrompt();
-   let cardPrompt = this.promptForCardGen;
-    let args = new GenerateArgs(generate_card, generate_typology, false, {
-      typology_prompt: typologyPrompt,
-      card_gen_prompt: cardPrompt,
-      summary_prompt: "",
-    });
-    const responseToReturn = [];
-    const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
-    for (let elem of whatNeedsToBeGenerated)
-      if (elem == "generate_tyopology") {
-        this.typologyResponse = await this.generateTypology(
-          args.prompts.typology_prompt ?? ""
-        );
-        responseToReturn.push(this.typologyResponse);
-      } else if (elem == "generate_card") {
-        if(this.typologyResponse.generate_cards){
-          if(this.typologyResponse.generate_cards.state == false){
-          console.log('Cards Generation Not Required');
-        }else{
-          this.cardgenResponse = await this.generateCard(
-            args.prompts.card_gen_prompt ?? "",
-            this.parsedContent + JSON.stringify(this.typologyResponse),
-            false
-          );
-        }
-      }
-
-        else{
-        this.cardgenResponse = await this.generateCard(
-          args.prompts.card_gen_prompt ?? "",
-          this.parsedContent,
-          false
-        );
-        
-      }
-      responseToReturn.push(this.cardgenResponse);
-    }
-    if (this.cardgenResponse.status_code == 200) {
-      let gapFill = gapFilling(this.typologyResponse, this.cardgenResponse);
-      if (
-        gapFill.remainingConcepts.length !== 0 ||
-        gapFill.remainingFacts.length !== 0
-      ) {
-        this.gapFillResponse = await this.generateCard(
-          args.prompts.card_gen_prompt ?? "",
-          this.parsedContent +
-            "Generate cards only suitable for the given remaining concepts and facts" +
-            JSON.stringify(gapFill) +
-            "Exclude generating these cards" +
-            JSON.stringify(this.cardgenResponse.cards_data),
-          true
-        );
-      }
-      responseToReturn.push(this.gapFillResponse);
-    }
- 
-    return responseToReturn;
-   // return [typologyPrompt, cardPrompt];
-  
-  }
-
-  async generateCard(prompt: string, content: string, isGapFill: boolean) {
-    let generateCards = new GenerateCards(this.openAiService);
-    let cardgenResponse = await generateCards.generateCards(
-      prompt ?? "",
-      content,
-      isGapFill
-    );
-    // let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-    // response['type'] = 'card_gen';
-    return cardgenResponse;
-  }
-
-  async generateTypology(prompt: string) {
-    let response = await new GenerateTypology(
-      this.openAiService,
-      prompt,
-      this.parsedContent,
-      this.expectedFields
-    ).generate();
-    return response;
-  }
-
-  
-
-}
+import { GenerateCards } from "../card_gen/generate_cards";
+import { ParseSourceContent } from "../parse/parse_source_content";
+import { OpenAiService } from "../services/open_ai_service";
+import { returnCardGenPrompt } from "../constants/prompts/card_gen_prompt";
+import { returnTypologyPrompt } from "../constants/prompts/typology_prompt";
+import { GenerateTypology } from "../typology_gen/generate_typology";
+import { GenerateArgs } from "../utils/generate_args";
+import { gapFilling } from "../gap_fill/calculate_gap_fill";
+
+
+/// OnlyEverGenerator
+
+export class OnlyEverGenerator {
+  public api_key: string = "";
+  public openAiService: OpenAiService;
+
+/// these fields will be populated inside the constructor
+  parsedContent: any = {};
+  promptForTypology: string = "";
+  promptForCardGen: string = "";
+  expectedFields: Array<string>;
+
+
+  typologyResponse: any = {};
+  cardgenResponse: any = {};
+  summarizeResponse = {};
+  gapFillResponse: any = {};
+
+
+  constructor(
+    apiKey: string,
+    model: string,
+    generationContent : any
+  ) {
+    this.api_key = apiKey;
+    this.openAiService = new OpenAiService(
+      apiKey,
+      model ?? "gpt-3.5-turbo-1106"
+    );
+    const parsedData = new ParseSourceContent(generationContent.content).parseData()
+    this.parsedContent = {
+      title: parsedData.title,
+      headings: parsedData.headings,
+      content: parsedData.content,
+
+    },
+    // parsedData.type == 'cards' ? this.typologyResponse = parsedData.taxonomy :  this.typologyResponse = null;
+    this.typologyResponse = parsedData.taxonomy
+    
+    this.expectedFields =  generationContent.content.fields; //returnFields();
+    this.promptForTypology = returnTypologyPrompt(generationContent.prompt.typology);
+    this.promptForCardGen = returnCardGenPrompt(generationContent.prompt.card_generation);
+  }
+
+
+ 
+  async generate(
+    generate_typology: boolean = false,
+    generate_card: boolean = false
+  ): Promise<Array<any>> {
+    let args = new GenerateArgs(generate_card, generate_typology, false, );
+    const responseToReturn = [];
+    const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
+    for (let elem of whatNeedsToBeGenerated)
+      if (elem == "generate_tyopology") {
+        this.typologyResponse = await this.generateTypology(
+        this.promptForTypology
+        );
+        responseToReturn.push(this.typologyResponse);
+      } else if (elem == "generate_card") {
+        /// for cards gen to occur, there must be presence of source taxonomy
+        if(this.shouldTheCardBeGeneratedAfterTypologyResponse()){
+          this.cardgenResponse =  await this.generateCard(
+            this.promptForCardGen,
+            JSON.stringify(this.typologyResponse),
+            false,
+          )
+          responseToReturn.push(this.cardgenResponse);
+
+          /// check if gap fill is required ie coverage determination 
+          if(this.cardgenResponse.status_code == 200) {
+            this.gapFillResponse = await this.generationForGapFill(this.typologyResponse, this.cardgenResponse);
+            responseToReturn.push(this.gapFillResponse);
+          }
+
+        }
+    }
+    return responseToReturn;
+   // return [typologyPrompt, cardPrompt];
+  
+  }
+
+  shouldTheCardBeGeneratedAfterTypologyResponse(){
+    if(this.typologyResponse){
+      return this.typologyResponse.generate_cards.state == true;
+    }else{
+      return false;
+    }
+
+  }
+
+  async generationForGapFill(typologyData: any, cardGenData: any) {
+      let gapFill = gapFilling(typologyData, cardGenData);
+      let response :any ;
+      if (
+        gapFill.remainingConcepts.length !== 0 ||
+        gapFill.remainingFacts.length !== 0
+      ) {
+        response = await this.generateCard(
+         this.promptForCardGen +
+            "Generate cards only suitable for the given remaining concepts and facts" +
+            JSON.stringify(gapFill) +
+            "Exclude generating these cards",
+            JSON.stringify(cardGenData.cards_data),
+          true
+        );
+      }
+      return response;
+    
+  }
+
+
+
+  async generateCard(prompt: string, additionalContent: string, isGapFill: boolean) {
+    let generateCards = new GenerateCards(this.openAiService);
+    this.cardgenResponse = await generateCards.generateCards(
+      prompt ?? "",
+      this.parsedContent + additionalContent,
+      isGapFill
+    );
+  
+    // let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
+    // response['type'] = 'card_gen';
+    return this.cardgenResponse;
+  }
+
+
+  async generateTypology(prompt: string) {
+    let response = await new GenerateTypology(
+      this.openAiService,
+      prompt,
+      JSON.stringify(this.parsedContent),
+      this.expectedFields
+    ).generate();
+    return response;
+  }
+
+  
+
+}