only_ever_generator 0.7.8 → 0.8.0

package/src/embedding_generation/consolidation/global_consolidation.ts

@@ -0,0 +1,94 @@
+import { getSimilarConcepts } from "../../helper/qdrant_db_methods";
+
+export class GlobalConsolidation {
+  async consolidate(
+    locally_consolidated_concepts_facts: {
+      text: string;
+      type: string;
+      reference: string;
+      id: string;
+      embedding: number[];
+    }[],
+    sourceId: string,
+    threshold: number
+  ) {
+    // / for all concepts_facts, find the ,most similar concepts_facts in qdrant, threshold of 0.8
+    const similarConcepts = await getSimilarConcepts(
+      "concepts-vectors",
+      locally_consolidated_concepts_facts,
+      threshold ?? 0.8
+    );
+    const taxonomyConcepts: {
+      text: string;
+      type: string;
+      reference: string;
+      id: string;
+    }[] = [];
+
+    const globalUpdatesOps: {
+      id: string;
+      sourceIdToAdd: string;
+      currentPayload: {
+        _sources: string[];
+        text: string;
+      };
+    }[] = [];
+
+    const globalInsertsOps: {
+      id: string;
+      vector: number[];
+      payload: {
+        _sources: string[];
+        text: string;
+      };
+    }[] = [];
+    for (const index in similarConcepts) {
+      const points = similarConcepts[index].points;
+      const originalConcept = locally_consolidated_concepts_facts[index];
+      if (points.length == 0) {
+        taxonomyConcepts.push({
+          id: originalConcept.id,
+          text: originalConcept.text,
+          type: originalConcept.type,
+          reference: originalConcept.reference,
+        });
+        globalInsertsOps.push({
+          id: originalConcept.id,
+          vector: originalConcept.embedding,
+          payload: {
+            _sources: [sourceId],
+            text: originalConcept.text,
+          },
+        });
+      } else {
+        const consolidatedId = points[0].id;
+        const currentPayload = points[0].payload;
+        // const currentSources = currentPayload?['_sources'] ?? [];
+        taxonomyConcepts.push({
+          id: consolidatedId.toString(),
+          text: originalConcept.text,
+          type: originalConcept.type,
+          reference: originalConcept.reference,
+        });
+        globalUpdatesOps.push({
+          id: consolidatedId.toString(),
+          sourceIdToAdd: sourceId,
+          currentPayload: {
+            // _sources: currentPayload['_sources'] ?? [],
+            _sources: (currentPayload?._sources ?? []) as string[],
+            text: (currentPayload?.text ?? "").toString(),
+          },
+        });
+      }
+    }
+    return {
+      source_taxonomy: taxonomyConcepts,
+      global_updates: globalUpdatesOps,
+      global_inserts: globalInsertsOps,
+    };
+    /// if threshold is below 0.8, then add the concept_fact to qdrant
+    /// if threshold is above 0.8, then replace the id of the local concept_fact with the id of the global concept_fact
+    /// add _source.id to qdrant
+    /// return the concepts_facts
+  }
+}

package/src/embedding_generation/consolidation/local_consolidation.ts

@@ -0,0 +1,141 @@
+import { randomUUID, UUID } from "crypto";
+
+export class LocalConsolidation {
+  consolidate(
+    concepts_facts: {
+      text: string;
+      type: string;
+      embedding: number[];
+      reference: string;
+    }[],
+    sourceId: string
+  ) {
+    try {
+      const sourceIds = [];
+      const sourceTaxonomyOps: {
+        text: string;
+        id: string;
+        embedding: number[];
+        type: string;
+        reference: string;
+      }[] = [];
+      const globalConceptOps: {
+        id: string;
+        vector: number[];
+        payload: {
+          _sources: string[];
+          text: string;
+        };
+      }[] = [];
+      const consolidatedConcepts: any[] = [];
+      const concepts: {
+        id: string;
+        text: string;
+        reference: string;
+        embedding: number[];
+        type: string;
+      }[] = concepts_facts.map((e: any) => {
+        return {
+          id: randomUUID().toString(),
+          text: e.text,
+          reference: e.reference,
+          embedding: e.embedding,
+          type: e.type,
+        };
+      });
+      const afterConsolidation = this.consolidateSimilarEmbeddings(concepts);
+      sourceIds.push(JSON.stringify(sourceId));
+      console.log(afterConsolidation);
+      for (var concept of afterConsolidation.finalConsolidatedConcepts) {
+        sourceTaxonomyOps.push({
+          text: concept.text,
+          id: concept.id,
+          embedding: concept.embedding,
+          type: concept.type,
+          reference: concept.reference,
+        });
+        globalConceptOps.push({
+          id: concept.id.toString(),
+          vector: concept.embedding,
+          payload: {
+            _sources: [sourceId],
+            text: concept.text,
+          },
+        });
+      }
+      consolidatedConcepts.push(...afterConsolidation.consolidatedConcepts);
+
+      return {
+        globalConceptOps: globalConceptOps,
+        sourceIds: sourceIds,
+        consolidatedConcepts: consolidatedConcepts,
+        sourceTaxonomyOps: sourceTaxonomyOps,
+      };
+    } catch (e: any) {
+      console.error(
+        "Error occurred while converting the parsed array to db operations:",
+        e
+      );
+      throw e;
+    }
+    // const concepts = concepts_facts.map((e) => {
+    //   return {
+    //     id: randomUUID().toString(),
+    //     text: e.text,
+    //     type: e.type,
+    //     embedding: e.embedding,
+    //   };
+    // });
+    // const consolidated = this._consolidateSimilarEmbeddings(concepts);
+    // return consolidated;
+  }
+
+  cosineSimilarity = (a: number[], b: number[]): number => {
+    const dotProduct = a.reduce((acc, val, index) => acc + val * b[index], 0);
+    const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
+    const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
+    return dotProduct / (magnitudeA * magnitudeB);
+  };
+
+  consolidateSimilarEmbeddings(data: any[], threshold = 0.65) {
+    const finalConsolidatedConcepts: {
+      id: UUID;
+      text: string;
+      reference: string;
+      embedding: number[];
+      type: string;
+    }[] = [];
+    const visited = new Array(data.length).fill(false);
+    const consolidatedConcepts: any[] = [];
+
+    for (let i = 0; i < data.length; i++) {
+      if (visited[i]) continue;
+
+      const group = [data[i]];
+      visited[i] = true;
+
+      for (let j = i + 1; j < data.length; j++) {
+        if (visited[j]) continue;
+
+        const sim = this.cosineSimilarity(data[i].embedding, data[j].embedding);
+        if (sim > threshold) {
+          group.push(data[j]);
+          visited[j] = true;
+          consolidatedConcepts.push({
+            concept1: data[i].text,
+            concept2: data[j].text,
+            similarity: sim,
+          });
+        }
+      }
+
+      // Consolidate the group (e.g., just take the first, or merge)
+      finalConsolidatedConcepts.push(group[0]); // Or you can customize how to merge
+    }
+
+    return {
+      finalConsolidatedConcepts: finalConsolidatedConcepts,
+      consolidatedConcepts: consolidatedConcepts,
+    };
+  }
+}

package/src/embedding_generation/consolidation/write_consolidated_data.ts

@@ -0,0 +1,98 @@
+import qdrantClient from "../../services/qdrant_service";
+
+export class WriteConsolidatedData {
+  async writeConsolidatedData(
+    data: {
+      source_taxonomy: {
+        id: string;
+        text: string;
+        type: string;
+        reference: string;
+      }[];
+      global_updates: {
+        id: string;
+        currentPayload: {
+          _sources: string[];
+          text: string;
+        };
+        sourceIdToAdd: string;
+      }[];
+      global_inserts: {
+        id: string;
+        vector: number[];
+        payload: {
+          _sources: string[];
+          text: string;
+        };
+      }[];
+    },
+    generation_requests: any[],
+    source_id: string
+  ) {
+    if (data.global_updates.length > 0) {
+      await this.writeGlobalUpdates(data.global_updates);
+    }
+    if (data.global_inserts.length > 0) {
+      await this.writeGlobalInserts(data.global_inserts);
+    }
+
+    return {
+      status: "success",
+    };
+  }
+
+  async writeGlobalUpdates(
+    data: {
+      id: string;
+      currentPayload: {
+        _sources: string[];
+        text: string;
+      };
+      sourceIdToAdd: string;
+    }[]
+  ) {
+    const operations = data.map((e) => {
+      return {
+        set_payload: {
+          points: [e.id],
+          payload: {
+            text: e.currentPayload.text,
+            _sources: [e.sourceIdToAdd, ...e.currentPayload._sources],
+          },
+        },
+      };
+    });
+    await qdrantClient.batchUpdate("concepts-vectors", {
+      wait: true,
+      operations: operations,
+    });
+  }
+
+  async writeGlobalInserts(
+    data: {
+      id: string;
+      vector: number[];
+      payload: {
+        _sources: string[];
+        text: string;
+      };
+    }[]
+  ) {
+    await qdrantClient.upsert("concepts-vectors", {
+      wait: true,
+      points: data,
+    });
+  }
+
+  async wiriteToMondo(
+    source_taxonomy: any[],
+    generation_requests: any[],
+    source_id: string
+  ) {
+    const mondo_data = source_taxonomy.map((e) => {
+      return {
+        id: e.id,
+      };
+    });
+  }
+}

package/src/embedding_generation/generate_embeddings.ts

@@ -0,0 +1,41 @@
+import { OpenAiService } from "../services/open_ai_service";
+
+export class GenerateEmbeddings {
+  constructor(private readonly openAiService: OpenAiService) {}
+
+  async generateEmbeddings(
+    concepts_facts: { text: string; type: string; reference: string }[]
+  ) {
+    const texts = concepts_facts.map((e: any) => e.text);
+    const response = await this.openAiService.sendEmbeddingRequest(texts);
+    if (response.status_code !== 200) {
+      throw new Error(response.message);
+    }
+    const embeddings = response.data.data.sort((a: any) => a.index);
+    const model = response.data.model;
+    const usage = response.data.usage;
+    const embeddings_map = [];
+    for (let i = 0; i < concepts_facts.length; i++) {
+      embeddings_map.push({
+        text: concepts_facts[i].text,
+        type: concepts_facts[i].type,
+        embedding: embeddings[i].embedding,
+        reference: concepts_facts[i].reference,
+      });
+    }
+    // return embeddings_map;
+    return {
+      concepts_facts: embeddings_map,
+      metadata: {
+        req_time: new Date().toISOString(),
+        req_type: {
+          type: "embedding",
+        },
+        req_tokens: usage.prompt_tokens,
+        res_tokens: usage.completion_tokens,
+        model: model,
+        usage: usage,
+      },
+    };
+  }
+}

package/src/embedding_generation/parse_embedding_response.ts

@@ -0,0 +1,31 @@
+import { LocalConsolidation } from "./consolidation/local_consolidation";
+
+export class ParseEmbeddingResponse {
+  parse(
+    response: any,
+    concepts_facts: { text: string; type: string; reference: string }[]
+  ) {
+    const data = response.data;
+    const model = data.model;
+    const usage = data.usage;
+    const rawEmbeddings = data.data;
+    const sorted = rawEmbeddings.sort((e: any) => e.index);
+    const embeddings = sorted.map((e: any) => e.embedding);
+    const embeddings_map = new Map();
+    const embeddings_map_array = [];
+    for (let i = 0; i < concepts_facts.length; i++) {
+      embeddings_map_array.push({
+        text: concepts_facts[i].text,
+        type: concepts_facts[i].type,
+        embedding: embeddings[i],
+        reference: concepts_facts[i].reference,
+      });
+    }
+    const embeddingMap = embeddings_map;
+    const consolidated = new LocalConsolidation().consolidate(
+      embeddings_map_array,
+      "sourceID"
+    );
+    return consolidated;
+  }
+}

package/src/enums/card_type_enum.ts

@@ -0,0 +1,6 @@
+export enum CardTypeEnum {
+  FLASH = "flash",
+  CLOZE = "cloze",
+  MATCH = "match",
+  MCQ = "mcq",
+}

package/src/gap_fill/calculate_gap_fill.ts

@@ -1,53 +1,50 @@
 function isEmpty(obj: object): boolean {
-    return Object.keys(obj).length === 0;
+  return Object.keys(obj).length === 0;
+}
+
+export function gapFilling(typologyResponse: any, cardgenResponse: any) {
+  let allConcepts: string[] = [];
+  let allFacts: string[] = [];
+  let generatedConceptsList: string[] = [];
+  let generatedFactsList: string[] = [];
+  let remainingConcepts: string[] = [];
+  let remainingFacts: string[] = [];
+  if (!isEmpty(typologyResponse)) {
+    allConcepts.push(...(typologyResponse.concepts ?? []));
+    allFacts.push(...(typologyResponse?.facts ?? []));
   }
 
+  if (!isEmpty(cardgenResponse)) {
+    allConcepts.push(...(cardgenResponse.missing_concepts ?? []));
+    allFacts.push(...(cardgenResponse.missing_facts ?? []));
+  }
 
-export function gapFilling(typologyResponse: any, cardgenResponse : any) {
-    let allConcepts: string[] = [];
-    let allFacts: string[] = [];
-    let generatedConceptsList: string[] = [];
-    let generatedFactsList: string[] = [];
-    let remainingConcepts: string[] = [];
-    let remainingFacts: string[] = [];
-    if (!isEmpty(typologyResponse)) {
-      allConcepts.push(...(typologyResponse.concepts ?? []));
-      allFacts.push(...(typologyResponse?.facts ?? []));
+  if (
+    cardgenResponse.cards_data !== undefined &&
+    cardgenResponse.cards_data?.length != 0
+  ) {
+    for (let card of cardgenResponse.cards_data) {
+      if (card.concepts.length != 0) {
+        generatedConceptsList.push(...card.concepts);
+      }
+      if (card.facts.length != 0) {
+        generatedFactsList.push(...card.facts);
+      }
     }
 
-    if (!isEmpty(cardgenResponse)) {
-      allConcepts.push(...(cardgenResponse.missing_concepts ?? []));
-      allFacts.push(...(cardgenResponse.missing_facts ?? []));
-    }
+    let generatedConceptsSet = Array.from(new Set(generatedConceptsList));
+    let generatedFactsSet = Array.from(new Set(generatedFactsList));
 
-    if(cardgenResponse.cards_data !== undefined && cardgenResponse.cards_data?.length != 0) {
-      for (let card of cardgenResponse.cards_data) {
-        if (card.concepts.length != 0) {
-          generatedConceptsList.push(...card.concepts);
-        }
-        if (card.facts.length != 0) {
-          generatedFactsList.push(...card.facts);
-        }
-      }
-  
-      let generatedConceptsSet = Array.from(new Set(generatedConceptsList));
-      let generatedFactsSet = Array.from(new Set(generatedFactsList));
-  
-      remainingConcepts = allConcepts.filter(
-        (item) => !generatedConceptsSet.includes(item)
-      );
-       remainingFacts = allFacts.filter(
-        (item) => !generatedFactsSet.includes(item)
-      );
-    }
-    
+    remainingConcepts = allConcepts.filter(
+      (item) => !generatedConceptsSet.includes(item)
+    );
+    remainingFacts = allFacts.filter(
+      (item) => !generatedFactsSet.includes(item)
+    );
+  }
 
-    return {
-      //   allConcepts: allConcepts,
-      //   allFacts: allFacts,
-      //   generatedConcepts: generatedConceptsSet,
-      //   generatedFacts: generatedFactsSet,
-      remainingConcepts: remainingConcepts,
-      remainingFacts: remainingFacts,
-    };
-  }
+  return {
+    remainingConcepts: remainingConcepts,
+    remainingFacts: remainingFacts,
+  };
+}

package/src/helper/qdrant_db_methods.ts

@@ -0,0 +1,77 @@
+import qdrantClient from "../services/qdrant_service";
+
+const createCollection = async (collectionName: string) => {
+  await qdrantClient.createCollection(collectionName, {
+    vectors: {
+      size: 1536,
+      distance: "Cosine",
+    },
+  });
+};
+
+const getCollection = async () => {
+  const collection = await qdrantClient.getCollections();
+  return collection;
+};
+
+const getCorrespondingConcepts = async (
+  collectionName: string,
+  embeddings: {
+    text: string;
+    id: string;
+    type: string;
+    reference: string;
+    embedding: number[];
+  }[],
+  threshold: number
+) => {
+  try {
+    const searchQuery = embeddings.map((e) => {
+      return {
+        query: e.embedding,
+        limit: 1,
+        score_threshold: threshold,
+        with_payload: true,
+      };
+    });
+    const results = await qdrantClient.queryBatch(collectionName, {
+      searches: searchQuery,
+    });
+    return results;
+  } catch (error) {
+    console.log(error);
+    throw error;
+  }
+};
+
+const addEmbeddingsToCollection = async (
+  collectionName: string,
+  embeddings: {
+    id: string;
+    vector: number[];
+    payload: {
+      _sources: string[];
+      text: string;
+    };
+  }[]
+) => {
+  const CHUNK_SIZE = 1000;
+  for (let i = 0; i < embeddings.length; i += CHUNK_SIZE) {
+    const batch = embeddings.slice(i, i + CHUNK_SIZE);
+    await qdrantClient.upsert(
+      collectionName,
+
+      {
+        wait: true,
+        points: batch,
+      }
+    );
+  }
+};
+
+export {
+  createCollection,
+  getCollection,
+  addEmbeddingsToCollection,
+  getCorrespondingConcepts as getSimilarConcepts,
+};

package/src/parse/parse_card/parse_cloze_card.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawClozeCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseClozeCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawClozeCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       const content = data.card_content;
       let correctOptions = content.correct_options;
@@ -36,9 +44,9 @@ export class ParseClozeCard {
           question: finalQuestion,
           options: finalParsedOptions,
         },
-        concepts: data.concepts,
-        facts: data.facts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
+        bloom_level: data.bloom_level,
       };
 
       return this._validateCloze(clozeCardData);

package/src/parse/parse_card/parse_flash_cards.ts

@@ -1,5 +1,13 @@
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
+import { RawFlashCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+
 export class ParseFlashCard {
-  parse(data: any) {
+  parse(data: {
+    card_content: RawFlashCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let displayTitle = this.generateFlashCardDisplayTitle(
         data.card_content.front,
@@ -8,7 +16,7 @@ export class ParseFlashCard {
       let flashCardData = {
         type: {
           category: "learning",
-          sub_type: data.type,
+          sub_type: "flash",
         },
         heading: "",
         displayTitle: displayTitle,
@@ -16,9 +24,9 @@ export class ParseFlashCard {
           front_content: data.card_content.front,
           back_content: data.card_content.back,
         },
-        concepts: data.concepts,
+        concepts_facts: data.concepts_facts,
         explanation: data.card_content.explanation,
-        facts: data.facts,
+        bloom_level: data.bloom_level,
       };
 
       return flashCardData;

package/src/parse/parse_card/parse_match_card.ts

@@ -1,4 +1,6 @@
 import { match } from "assert";
+import { RawMatchCardResponseType } from "../../types/raw_card_response_types/generated_card_response_type";
+import { MongoConceptFactCards } from "../../types/mongo_concept_fact_type";
 
 type InputItem = {
   left_item: string;
@@ -11,7 +13,12 @@ type OutputItem = {
 };
 
 export class ParseMatchCard {
-  parse(cardData: any) {
+  parse(cardData: {
+    card_content: RawMatchCardResponseType;
+    type: string;
+    concepts_facts: MongoConceptFactCards[];
+    bloom_level: number;
+  }) {
     try {
       let content = cardData.card_content;
       const finalContent = this._parseMatchContent(content);
@@ -26,9 +33,8 @@ export class ParseMatchCard {
         content: finalContent,
         //  content: cardData.card_content,
         displayTitle: displayTitle,
-        concepts: cardData.concepts,
-        facts: cardData.facts,
-        explanation: cardData.card_content.explanation,
+        concepts_facts: cardData.concepts_facts,
+        bloom_level: cardData.bloom_level,
       };
 
       return this._validateMatch(matchCard);