only_ever_generator 0.7.8 → 0.7.9

package/dist/types/source_taxonomy_type.d.ts

@@ -0,0 +1,14 @@
+export type SourceTaxonomy = {
+    concepts_facts: {
+        text: string;
+        type: string;
+        embedding?: number[];
+        reference: string;
+    }[];
+    fields: string[];
+    metadata?: any[];
+    generate_cards: {
+        state: boolean;
+        reason: string;
+    };
+};

package/dist/types/source_taxonomy_type.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=source_taxonomy_type.js.map

package/dist/types/source_taxonomy_type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source_taxonomy_type.js","sourceRoot":"","sources":["../../src/types/source_taxonomy_type.ts"],"names":[],"mappings":""}

package/dist/typology_gen/generate_typology.d.ts

@@ -0,0 +1,28 @@
+import { OpenAiService } from "../services/open_ai_service";
+export declare class GenerateTypology {
+    openAiService: OpenAiService;
+    prompt: string;
+    content: string;
+    expectedFields: Array<string>;
+    constructor(openAiService: OpenAiService, prompt: string, content: string, expected_fields: Array<string>);
+    generate(): Promise<any>;
+    parseTypologyOnSuccess(responseData: any): {
+        status_code: number;
+        metadata: any[];
+        field: {
+            [x: string]: string | boolean;
+            reconcile: boolean;
+        }[];
+        concepts_facts: any[];
+        generate_cards: any;
+        summary_cards: any;
+    };
+    parseFields(fields: Array<string>): {
+        [x: string]: string | boolean;
+        reconcile: boolean;
+    }[];
+    parseTypologyOnFailure(responseData: any): Promise<{
+        status_code: any;
+        metadata: any;
+    }>;
+}

package/dist/typology_gen/generate_typology.js

@@ -58,26 +58,26 @@ class GenerateTypology {
     }
     parseTypologyOnSuccess(responseData) {
         responseData.metadata.status = "completed";
-        const facts = responseData.generated_content.facts.map((fact) => {
+        const generatedContent = responseData.generated_content;
+        const concepts = generatedContent.concepts.map((e) => {
             return {
-                concept_text: fact.fact_text,
-                reference: fact.reference,
-                type: "fact",
+                text: e.concept_text,
+                type: "concept",
+                reference: e.reference,
             };
         });
-        const concepts = responseData.generated_content.concepts.map((concept) => {
+        const facts = generatedContent.facts.map((e) => {
             return {
-                concept_text: concept.concept_text,
-                reference: concept.reference,
-                type: "concept",
+                text: e.fact_text,
+                type: "fact",
+                reference: e.reference,
             };
         });
-        const generatedContent = responseData.generated_content;
         return {
             status_code: 200,
-            metadata: responseData.metadata,
+            metadata: [responseData.metadata],
             field: this.parseFields(generatedContent.field),
-            concepts: [...concepts, ...facts],
+            concepts_facts: [...concepts, ...facts],
             generate_cards: generatedContent.generate_cards,
             summary_cards: generatedContent.summary_cards,
             // type: responseData.type,
@@ -101,3 +101,4 @@ class GenerateTypology {
     }
 }
 exports.GenerateTypology = GenerateTypology;
+//# sourceMappingURL=generate_typology.js.map

package/dist/typology_gen/generate_typology.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate_typology.js","sourceRoot":"","sources":["../../src/typology_gen/generate_typology.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,sCAAwC;AAExC,MAAa,gBAAgB;IAK3B,YACE,aAA4B,EAC5B,MAAc,EACd,OAAe,EACf,eAA8B;QAPzB,WAAM,GAAW,EAAE,CAAC;QACpB,YAAO,GAAW,EAAE,CAAC;QAQ1B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE,CACzD,IAAI,CAAC,WAAW,EAAE,CACnB,CAAC;IACJ,CAAC;IACK,QAAQ;;;YACZ,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,CAAA,MAAA,IAAI,CAAC,aAAa,0CAAE,WAAW,CACpD,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb,CAAA,CAAC;gBACF,QAAQ,CAAC,cAAc,CAAC,GAAG;oBACzB,IAAI,EAAE,SAAS;oBACf,CAAC,EAAE,CAAC;iBACL,CAAC;gBACF,QAAQ,CAAC,QAAQ,GAAG;oBAClB,QAAQ,EAAE,MAAA,QAAQ,CAAC,YAAY,mCAAI,IAAI,IAAI,EAAE;oBAC7C,QAAQ,EAAE;wBACR,IAAI,EAAE,SAAS;wBACf,CAAC,EAAE,CAAC;qBACL;oBACD,UAAU,EAAE,MAAA,QAAQ,CAAC,UAAU,0CAAE,aAAa;oBAC9C,UAAU,EAAE,MAAA,QAAQ,CAAC,UAAU,0CAAE,iBAAiB;oBAClD,qBAAqB,EAAE,MAAA,QAAQ,CAAC,UAAU,0CAAE,qBAAqB;oBACjE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;iBAChC,CAAC;gBACF,IAAI,QAAQ,CAAC,WAAW,IAAI,GAAG,EAAE,CAAC;oBAChC,OAAO,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC;oBACjD,OAAO,QAAQ,CAAC;gBAClB,CAAC;YACH,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,MAAM,IAAI,oBAAW,CAAC;oBACpB,IAAI,EAAE,kBAAkB;oBACxB,IAAI,EAAE,CAAC,CAAC,OAAO;iBAChB,CAAC,CAAC,GAAG,EAAE,CAAC;YACX,CAAC;QACH,CAAC;KAAA;IAED,sBAAsB,CAAC,YAAiB;QACtC,YAAY,CAAC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;QAC3C,MAAM,gBAAgB,GAAG,YAAY,CAAC,iBAAiB,CAAC;QACxD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;YACxD,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,YAAY;gBACpB,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;YAClD,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,WAAW,EAAE,GAAG;YAChB,QAAQ,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC;YACjC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC;YAC/C,cAAc,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAC;YACvC,cAAc,EAAE,gBAAgB,CAAC,cAAc;YAC/C,aAAa,EAAE,gBAAgB,CAAC,aAAa;YAC7C,2BAA2B;SAC5B,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,MAAqB;QAC/B,MAAM,SAAS,GAAG,CAAC,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACzE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9C,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;SAC7D,CAAC,CAAC,CAAC;IACN,CAAC;IAEK,sBAAsB,CAAC,YAAiB;;YAC5C,YAAY,CAAC,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC;YAExC,OAAO;gBACL,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,QAAQ,EAAE,YAAY,CAAC,QAAQ;aAChC,CAAC;QACJ,CAAC;KAAA;CACF;AAlGD,4CAkGC"}

package/dist/utils/generate_args.d.ts

@@ -0,0 +1,7 @@
+export declare class GenerateArgs {
+    generate_card: boolean;
+    generate_typology: boolean;
+    generate_summary: boolean;
+    constructor(generate_card: boolean, generate_typology: boolean, generate_summary: boolean);
+    getWhatNeedsToBeGenerated(): string[];
+}

package/dist/utils/generate_args.js

@@ -25,3 +25,4 @@ class GenerateArgs {
     }
 }
 exports.GenerateArgs = GenerateArgs;
+//# sourceMappingURL=generate_args.js.map

package/dist/utils/generate_args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate_args.js","sourceRoot":"","sources":["../../src/utils/generate_args.ts"],"names":[],"mappings":";;;AAAA,MAAa,YAAY;IAKvB,YACE,aAAsB,EACtB,iBAA0B,EAC1B,gBAAyB;QAPpB,kBAAa,GAAY,KAAK,CAAC;QAC/B,sBAAiB,GAAY,KAAK,CAAC;QACnC,qBAAgB,GAAY,KAAK,CAAC;QAOvC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC3C,CAAC;IAED,yBAAyB;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACnC,UAAU,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AA5BD,oCA4BC"}

package/dist/utils/parse_openai_response.d.ts

@@ -0,0 +1,10 @@
+export declare function parseOpenAiSuccessResponse(responseData: any): {
+    status_code: number;
+    usage_data: any;
+    generated_content: any;
+    generated_at: Date;
+};
+export declare function parseOpenAiFailureResponse(errorResponse: any): {
+    status_code: any;
+    message: any;
+};

package/dist/utils/parse_openai_response.js

@@ -21,3 +21,4 @@ function parseOpenAiFailureResponse(errorResponse) {
         message: (_b = (_a = errorResponse.data) === null || _a === void 0 ? void 0 : _a.error) === null || _b === void 0 ? void 0 : _b.code,
     };
 }
+//# sourceMappingURL=parse_openai_response.js.map

package/dist/utils/parse_openai_response.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse_openai_response.js","sourceRoot":"","sources":["../../src/utils/parse_openai_response.ts"],"names":[],"mappings":";;AAAA,gEAUC;AAED,gEAMC;AAlBD,SAAgB,0BAA0B,CAAC,YAAiB;IAC1D,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC;IAChC,IAAI,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC;IACvC,OAAO;QACL,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,MAAM;QAClB,iBAAiB,EAAE,OAAO;QAC1B,YAAY,EAAE,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;KAC3C,CAAC;AACJ,CAAC;AAED,SAAgB,0BAA0B,CAAC,aAAkB;;IAC3D,mBAAmB;IACnB,OAAO;QACL,WAAW,EAAE,aAAa,CAAC,MAAM;QACjC,OAAO,EAAE,MAAA,MAAA,aAAa,CAAC,IAAI,0CAAE,KAAK,0CAAE,IAAI;KACzC,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,27 +1,30 @@
 {
   "name": "only_ever_generator",
-  "version": "0.7.8",
-  "main": "dist/index.js",
+  "version": "0.7.9",
+  "main": "dist/app.js",
   "scripts": {
-    "start": "npm run build && nodemon dist/index.js",
+    "start": "npm run build && nodemon dist/app.js",
     "build": "tsc",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "example": "npm run build && nodemon dist/app.js"
   },
   "keywords": [],
-  "author": "shree",
+  "author": "",
   "license": "ISC",
+  "description": "",
   "devDependencies": {
-    "@types/express": "^4.17.21",
-    "@types/node": "^20.14.2",
-    "dotenv": "^16.4.5",
-    "express": "^4.19.2",
-    "nodemon": "^3.1.3",
+    "@types/express": "^5.0.1",
+    "@types/node": "^22.15.3",
+    "dotenv": "^16.5.0",
+    "express": "^5.1.0",
+    "nodemon": "^3.1.10",
     "ts-node": "^10.9.2",
-    "typescript": "^5.5.4"
+    "typescript": "^5.8.3"
   },
   "dependencies": {
-    "axios": "^1.7.7",
-    "n": "^10.0.0",
+    "@qdrant/js-client-rest": "^1.14.0",
+    "axios": "^1.9.0",
+    "n": "^10.1.0",
     "stable": "^0.1.8"
   },
   "eslintConfig": {
@@ -30,4 +33,4 @@
       "typescript"
     ]
   }
-}
+}

package/src/bootstrap/app.ts

@@ -1,30 +1,35 @@
 import { GenerateCards } from "../card_gen/generate_cards";
 import { ParseSourceContent } from "../parse/parse_source_content";
 import { OpenAiService } from "../services/open_ai_service";
-// import { returnCardGenPrompt } from "../constants/prompts/card_gen_prompt";
-// import { returnTypologyPrompt } from "../constants/prompts/typology_prompt";
 import { GenerateTypology } from "../typology_gen/generate_typology";
 import { GenerateArgs } from "../utils/generate_args";
-import { gapFilling } from "../gap_fill/calculate_gap_fill";
-
-/// OnlyEverGenerator
+import { GenerateEmbeddings } from "../embedding_generation/generate_embeddings";
+import { LocalConsolidation } from "../embedding_generation/consolidation/local_consolidation";
+import { GlobalConsolidation } from "../embedding_generation/consolidation/global_consolidation";
+import { WriteConsolidatedData } from "../embedding_generation/consolidation/write_consolidated_data";
+import { BaseParamType } from "../types/base_param_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+import { MongoConceptFactSource } from "../types/mongo_concept_fact_type";
 
 export class OnlyEverGenerator {
   public api_key: string = "";
   public openAiService: OpenAiService;
 
   /// these fields will be populated inside the constructor
-  parsedContent: any = {};
+  parsedContent?: {
+    title: string;
+    headings: string[];
+    content: any[];
+    taxonomy: SourceTaxonomy;
+  } = undefined;
   promptForTypology: string = "";
   promptForCardGen: string = "";
-  expectedFields: Array<string>;
-
-  typologyResponse: any = undefined;
+  expectedFields: Array<string> = [];
+  typologyResponse?: SourceTaxonomy = undefined;
   cardgenResponse: any = undefined;
   summarizeResponse = {};
-  gapFillResponse: any = {};
 
-  constructor(apiKey: string, model: string, generationContent: any) {
+  constructor(apiKey: string, model: string, generationContent: BaseParamType) {
     this.api_key = apiKey;
     this.openAiService = new OpenAiService(
       apiKey,
@@ -39,7 +44,6 @@ export class OnlyEverGenerator {
       content: parsedData.content,
       taxonomy: parsedData.taxonomy,
     }),
-      // parsedData.type == 'cards' ? this.typologyResponse = parsedData.taxonomy :  this.typologyResponse = null;
       (this.typologyResponse = generationContent.content.taxonomy);
 
     this.expectedFields = generationContent.content.fields; //returnFields();
@@ -59,14 +63,35 @@ export class OnlyEverGenerator {
         this.typologyResponse = await this.generateTypology(
           this.promptForTypology
         );
+
+        const embeddings = await this.generateEmbeddings(this.typologyResponse);
+        console.log("Embedding generated");
+        const localConsolidation = await this.localConsolidation(
+          embeddings.concepts_facts,
+          this.parsedContent!.title
+        );
+        console.log("Local consolidation done");
+        this.typologyResponse!.concepts_facts =
+          localConsolidation.sourceTaxonomyOps;
+        const globalConsolidation = await this.globalConsolidation(
+          localConsolidation.sourceTaxonomyOps,
+          this.parsedContent!.title,
+          0.8
+        );
+        console.log("Global consolidation done");
+        this.typologyResponse!.concepts_facts =
+          globalConsolidation.source_taxonomy;
+        this.typologyResponse!.metadata?.push(embeddings.metadata);
         responseToReturn.push(this.typologyResponse);
       } else if (elem == "generate_card") {
-        /// for cards gen to occur, there must be presence of source taxonomy
         if (this.shouldTheCardBeGeneratedAfterTypologyResponse()) {
-          this.parsedContent.taxonomy = {
-            concepts: this.typologyResponse.concepts,
-            facts: this.typologyResponse.facts,
-            generate_cards: this.typologyResponse.generate_cards,
+          this.parsedContent!.taxonomy = {
+            concepts_facts: this.typologyResponse?.concepts_facts ?? [],
+            fields: this.expectedFields,
+            generate_cards: this.typologyResponse?.generate_cards ?? {
+              state: false,
+              reason: "",
+            },
           };
           this.cardgenResponse = await this.generateCard(
             this.promptForCardGen,
@@ -74,19 +99,9 @@ export class OnlyEverGenerator {
             false
           );
           responseToReturn.push(this.cardgenResponse);
-
-          /// check if gap fill is required ie coverage determination
-          if (this.cardgenResponse.status_code == 200) {
-            this.gapFillResponse = await this._generationForGapFill(
-              this.typologyResponse,
-              this.cardgenResponse
-            );
-            responseToReturn.push(this.gapFillResponse);
-          }
         }
       }
     return responseToReturn;
-    // return [typologyPrompt, cardPrompt];
   }
 
   shouldTheCardBeGeneratedAfterTypologyResponse() {
@@ -97,33 +112,11 @@ export class OnlyEverGenerator {
     }
   }
 
-  async _generationForGapFill(typologyData: any, cardGenData: any,) {
-    let gapFill = gapFilling(typologyData, cardGenData);
-    let response: any;
-    if (
-      gapFill.remainingConcepts.length !== 0 ||
-      gapFill.remainingFacts.length !== 0
-    ) {
-      this.typologyResponse.facts = gapFill.remainingFacts;
-      this.typologyResponse.concepts = gapFill.remainingConcepts;
-      response = await this.generateCard(
-        this.promptForCardGen +
-        "Generate cards only suitable for the given remaining concepts and facts" +
-        JSON.stringify(gapFill),
-
-        "",
-        true,
-        1,
-      );
-    }
-    return response;
-  }
-
   async generateCard(
     prompt: string,
     additionalContent: string,
     isGapFill: boolean,
-    n: number = 0,
+    n: number = 0
   ) {
     let generateCardsResp = await new GenerateCards(
       this.openAiService
@@ -131,11 +124,9 @@ export class OnlyEverGenerator {
       prompt ?? "",
       JSON.stringify(this.parsedContent) + additionalContent,
       isGapFill,
-      this.parsedContent.taxonomy, n,
+      this.parsedContent!.taxonomy,
+      n
     );
-
-    // let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
-    // response['type'] = 'card_gen';
     return generateCardsResp;
   }
 
@@ -149,26 +140,48 @@ export class OnlyEverGenerator {
     return response;
   }
 
-  async gapFill(factsMaps: any, aiCards: Array<any>, n: number = 0) {
-    /// factsmap
-    /// {
-    /// remaining_facts: [],
-    /// remaining_concepts: [],
-    //}
+  async generateEmbeddings(typologyResponse: any) {
+    const concepts_facts = typologyResponse.concepts_facts;
+    const texts = concepts_facts.map((e: MongoConceptFactSource) => e.text);
+    const embeddings = await new GenerateEmbeddings(
+      this.openAiService
+    ).generateEmbeddings(concepts_facts);
+    return embeddings;
+  }
 
-    /// aicards is data
-    let response: any;
+  async localConsolidation(
+    concepts_facts: {
+      text: string;
+      type: string;
+      reference: string;
+      embedding: number[];
+    }[],
+    sourceId: string
+  ) {
+    const embeddings = new LocalConsolidation().consolidate(
+      concepts_facts,
+      sourceId
+    );
+    return embeddings;
+  }
 
-    response = await this.generateCard(
-      this.promptForCardGen +
-      "Generate cards only suitable for the given remaining concepts and facts" +
-      JSON.stringify(factsMaps) +
-      "Exclude generating  cards with content in the following",
-      JSON.stringify(aiCards),
-      true,
-      n,
+  async globalConsolidation(
+    concepts_facts: {
+      id: string;
+      text: string;
+      type: string;
+      reference: string;
+      embedding: number[];
+    }[],
+    sourceId: string,
+    threshold: number
+  ) {
+    const embeddings = await new GlobalConsolidation().consolidate(
+      concepts_facts,
+      sourceId,
+      threshold ?? 0.8
     );
 
-    return response;
+    return embeddings;
   }
 }

package/src/card_gen/generate_cards.ts

@@ -1,6 +1,7 @@
 import { ErrorLogger } from "../logger";
 import { ParseCardResponse } from "../parse/parse_card_response";
 import { OpenAiService } from "../services/open_ai_service";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
 
 export class GenerateCards {
   openAiService: OpenAiService;
@@ -12,37 +13,47 @@ export class GenerateCards {
     prompt: string,
     parsedContent: string,
     isGapFill: boolean,
-    taxonomy: any,
-    n: number = 0,
+    taxonomy: SourceTaxonomy,
+    n: number = 0
   ) {
-    let response = await this.openAiService?.sendRequest(prompt, parsedContent);
-    var updatedNumber = n + 1;
-    // console.log("response to card generation ", response);
-    // response["request_type"] = ;
-    response.metadata = {
-      req_time: response.generated_at ?? new Date(),
-      req_type: {
-        type: "depth",
-        n: updatedNumber,
-        bloom_level: 1,
-      },
-      req_tokens: response.usage_data?.prompt_tokens,
-      res_tokens: response.usage_data?.completion_tokens,
-      prompt_tokens_details: response.usage_data?.prompt_tokens_details,
-      model: this.openAiService.model,
-    };
-    if (response.status_code == 200) {
-      response.metadata.status = "completed";
-      let parseCard = new ParseCardResponse().parse(
-        response,
-        isGapFill,
-        taxonomy
+    try {
+      let response = await this.openAiService?.sendRequest(
+        prompt,
+        parsedContent
       );
-      return parseCard;
-    } else {
-      response.metadata.status = "failed";
-      response.metadata.err_message = response.message;
-      return response;
+      var updatedNumber = n + 1;
+      response.metadata = {
+        req_time: response.generated_at ?? new Date(),
+        req_type: {
+          type: "depth",
+          n: updatedNumber,
+          bloom_level: 1,
+        },
+        req_tokens: response.usage_data?.prompt_tokens,
+        res_tokens: response.usage_data?.completion_tokens,
+        prompt_tokens_details: response.usage_data?.prompt_tokens_details,
+        model: this.openAiService.model,
+      };
+
+      if (response.status_code == 200) {
+        response.metadata.status = "completed";
+        let parseCard = new ParseCardResponse().parse(
+          response,
+          isGapFill,
+          taxonomy,
+          1
+        );
+        return parseCard;
+      } else {
+        response.metadata.status = "failed";
+        response.metadata.err_message = response.message;
+        return response;
+      }
+    } catch (e: any) {
+      new ErrorLogger({
+        type: "card_generation",
+        data: e.message,
+      }).log();
     }
   }
 }

package/src/config.ts

@@ -2,6 +2,8 @@ import "dotenv/config";
 
 const config = {
   openAIKey: process.env.OPEN_AI_KEY as string,
+  qdrantApiKey: process.env.QDRANT_API_KEY,
+  qdrantUrl: process.env.QDRANT_URL,
 };
 
 export default config;

package/src/constants/api_constants.ts

@@ -1,3 +1,7 @@
 export function openAiEndPoint() {
   return "https://api.openai.com/v1/chat/completions";
 }
+
+export function openAiEmbeddingEndPoint() {
+  return "https://api.openai.com/v1/embeddings";
+}

package/src/constants/prompts/card_gen_prompt.ts

@@ -160,5 +160,5 @@ Once you are done generating the test cards, review the full list of concepts an
 Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.
 `;
 export function returnCardGenPrompt() {
-    return promptString;
+  return promptString;
 }

package/src/embedding_generation/consolidation/global_consolidation.ts

@@ -0,0 +1,94 @@
+import { getSimilarConcepts } from "../../helper/qdrant_db_methods";
+
+export class GlobalConsolidation {
+  async consolidate(
+    locally_consolidated_concepts_facts: {
+      text: string;
+      type: string;
+      reference: string;
+      id: string;
+      embedding: number[];
+    }[],
+    sourceId: string,
+    threshold: number
+  ) {
+    // / for all concepts_facts, find the ,most similar concepts_facts in qdrant, threshold of 0.8
+    const similarConcepts = await getSimilarConcepts(
+      "concepts-vectors",
+      locally_consolidated_concepts_facts,
+      threshold ?? 0.8
+    );
+    const taxonomyConcepts: {
+      text: string;
+      type: string;
+      reference: string;
+      id: string;
+    }[] = [];
+
+    const globalUpdatesOps: {
+      id: string;
+      sourceIdToAdd: string;
+      currentPayload: {
+        _sources: string[];
+        text: string;
+      };
+    }[] = [];
+
+    const globalInsertsOps: {
+      id: string;
+      vector: number[];
+      payload: {
+        _sources: string[];
+        text: string;
+      };
+    }[] = [];
+    for (const index in similarConcepts) {
+      const points = similarConcepts[index].points;
+      const originalConcept = locally_consolidated_concepts_facts[index];
+      if (points.length == 0) {
+        taxonomyConcepts.push({
+          id: originalConcept.id,
+          text: originalConcept.text,
+          type: originalConcept.type,
+          reference: originalConcept.reference,
+        });
+        globalInsertsOps.push({
+          id: originalConcept.id,
+          vector: originalConcept.embedding,
+          payload: {
+            _sources: [sourceId],
+            text: originalConcept.text,
+          },
+        });
+      } else {
+        const consolidatedId = points[0].id;
+        const currentPayload = points[0].payload;
+        // const currentSources = currentPayload?['_sources'] ?? [];
+        taxonomyConcepts.push({
+          id: consolidatedId.toString(),
+          text: originalConcept.text,
+          type: originalConcept.type,
+          reference: originalConcept.reference,
+        });
+        globalUpdatesOps.push({
+          id: consolidatedId.toString(),
+          sourceIdToAdd: sourceId,
+          currentPayload: {
+            // _sources: currentPayload['_sources'] ?? [],
+            _sources: (currentPayload?._sources ?? []) as string[],
+            text: (currentPayload?.text ?? "").toString(),
+          },
+        });
+      }
+    }
+    return {
+      source_taxonomy: taxonomyConcepts,
+      global_updates: globalUpdatesOps,
+      global_inserts: globalInsertsOps,
+    };
+    /// if threshold is below 0.8, then add the concept_fact to qdrant
+    /// if threshold is above 0.8, then replace the id of the local concept_fact with the id of the global concept_fact
+    /// add _source.id to qdrant
+    /// return the concepts_facts
+  }
+}