only_ever_generator 8.4.6 → 8.4.7

package/src/embedding_generation/consolidation/local_consolidation.ts

@@ -1,141 +1,141 @@
-import { randomUUID, UUID } from "crypto";
-
-export class LocalConsolidation {
-  consolidate(
-    concepts_facts: {
-      text: string;
-      type: string;
-      embedding: number[];
-      reference: string;
-    }[],
-    sourceId: string
-  ) {
-    try {
-      const sourceIds = [];
-      const sourceTaxonomyOps: {
-        text: string;
-        id: string;
-        embedding: number[];
-        type: string;
-        reference: string;
-      }[] = [];
-      const globalConceptOps: {
-        id: string;
-        vector: number[];
-        payload: {
-          _sources: string[];
-          text: string;
-        };
-      }[] = [];
-      const consolidatedConcepts: any[] = [];
-      const concepts: {
-        id: string;
-        text: string;
-        reference: string;
-        embedding: number[];
-        type: string;
-      }[] = concepts_facts.map((e: any) => {
-        return {
-          id: randomUUID().toString(),
-          text: e.text,
-          reference: e.reference,
-          embedding: e.embedding,
-          type: e.type,
-        };
-      });
-      const afterConsolidation = this.consolidateSimilarEmbeddings(concepts);
-      sourceIds.push(JSON.stringify(sourceId));
-      console.log(afterConsolidation);
-      for (var concept of afterConsolidation.finalConsolidatedConcepts) {
-        sourceTaxonomyOps.push({
-          text: concept.text,
-          id: concept.id,
-          embedding: concept.embedding,
-          type: concept.type,
-          reference: concept.reference,
-        });
-        globalConceptOps.push({
-          id: concept.id.toString(),
-          vector: concept.embedding,
-          payload: {
-            _sources: [sourceId],
-            text: concept.text,
-          },
-        });
-      }
-      consolidatedConcepts.push(...afterConsolidation.consolidatedConcepts);
-
-      return {
-        globalConceptOps: globalConceptOps,
-        sourceIds: sourceIds,
-        consolidatedConcepts: consolidatedConcepts,
-        sourceTaxonomyOps: sourceTaxonomyOps,
-      };
-    } catch (e: any) {
-      console.error(
-        "Error occurred while converting the parsed array to db operations:",
-        e
-      );
-      throw e;
-    }
-    // const concepts = concepts_facts.map((e) => {
-    //   return {
-    //     id: randomUUID().toString(),
-    //     text: e.text,
-    //     type: e.type,
-    //     embedding: e.embedding,
-    //   };
-    // });
-    // const consolidated = this._consolidateSimilarEmbeddings(concepts);
-    // return consolidated;
-  }
-
-  cosineSimilarity = (a: number[], b: number[]): number => {
-    const dotProduct = a.reduce((acc, val, index) => acc + val * b[index], 0);
-    const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
-    const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
-    return dotProduct / (magnitudeA * magnitudeB);
-  };
-
-  consolidateSimilarEmbeddings(data: any[], threshold = 0.65) {
-    const finalConsolidatedConcepts: {
-      id: UUID;
-      text: string;
-      reference: string;
-      embedding: number[];
-      type: string;
-    }[] = [];
-    const visited = new Array(data.length).fill(false);
-    const consolidatedConcepts: any[] = [];
-
-    for (let i = 0; i < data.length; i++) {
-      if (visited[i]) continue;
-
-      const group = [data[i]];
-      visited[i] = true;
-
-      for (let j = i + 1; j < data.length; j++) {
-        if (visited[j]) continue;
-
-        const sim = this.cosineSimilarity(data[i].embedding, data[j].embedding);
-        if (sim > threshold) {
-          group.push(data[j]);
-          visited[j] = true;
-          consolidatedConcepts.push({
-            concept1: data[i].text,
-            concept2: data[j].text,
-            similarity: sim,
-          });
-        }
-      }
-
-      // Consolidate the group (e.g., just take the first, or merge)
-      finalConsolidatedConcepts.push(group[0]); // Or you can customize how to merge
-    }
-
-    return {
-      finalConsolidatedConcepts: finalConsolidatedConcepts,
-      consolidatedConcepts: consolidatedConcepts,
-    };
-  }
-}
+import { randomUUID, UUID } from "crypto";
+
+export class LocalConsolidation {
+  consolidate(
+    concepts_facts: {
+      text: string;
+      type: string;
+      embedding: number[];
+      reference: string;
+    }[],
+    sourceId: string
+  ) {
+    try {
+      const sourceIds = [];
+      const sourceTaxonomyOps: {
+        text: string;
+        id: string;
+        embedding: number[];
+        type: string;
+        reference: string;
+      }[] = [];
+      const globalConceptOps: {
+        id: string;
+        vector: number[];
+        payload: {
+          _sources: string[];
+          text: string;
+        };
+      }[] = [];
+      const consolidatedConcepts: any[] = [];
+      const concepts: {
+        id: string;
+        text: string;
+        reference: string;
+        embedding: number[];
+        type: string;
+      }[] = concepts_facts.map((e: any) => {
+        return {
+          id: randomUUID().toString(),
+          text: e.text,
+          reference: e.reference,
+          embedding: e.embedding,
+          type: e.type,
+        };
+      });
+      const afterConsolidation = this.consolidateSimilarEmbeddings(concepts);
+      sourceIds.push(JSON.stringify(sourceId));
+      console.log(afterConsolidation);
+      for (var concept of afterConsolidation.finalConsolidatedConcepts) {
+        sourceTaxonomyOps.push({
+          text: concept.text,
+          id: concept.id,
+          embedding: concept.embedding,
+          type: concept.type,
+          reference: concept.reference,
+        });
+        globalConceptOps.push({
+          id: concept.id.toString(),
+          vector: concept.embedding,
+          payload: {
+            _sources: [sourceId],
+            text: concept.text,
+          },
+        });
+      }
+      consolidatedConcepts.push(...afterConsolidation.consolidatedConcepts);
+
+      return {
+        globalConceptOps: globalConceptOps,
+        sourceIds: sourceIds,
+        consolidatedConcepts: consolidatedConcepts,
+        sourceTaxonomyOps: sourceTaxonomyOps,
+      };
+    } catch (e: any) {
+      console.error(
+        "Error occurred while converting the parsed array to db operations:",
+        e
+      );
+      throw e;
+    }
+    // const concepts = concepts_facts.map((e) => {
+    //   return {
+    //     id: randomUUID().toString(),
+    //     text: e.text,
+    //     type: e.type,
+    //     embedding: e.embedding,
+    //   };
+    // });
+    // const consolidated = this._consolidateSimilarEmbeddings(concepts);
+    // return consolidated;
+  }
+
+  cosineSimilarity = (a: number[], b: number[]): number => {
+    const dotProduct = a.reduce((acc, val, index) => acc + val * b[index], 0);
+    const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
+    const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
+    return dotProduct / (magnitudeA * magnitudeB);
+  };
+
+  consolidateSimilarEmbeddings(data: any[], threshold = 0.65) {
+    const finalConsolidatedConcepts: {
+      id: UUID;
+      text: string;
+      reference: string;
+      embedding: number[];
+      type: string;
+    }[] = [];
+    const visited = new Array(data.length).fill(false);
+    const consolidatedConcepts: any[] = [];
+
+    for (let i = 0; i < data.length; i++) {
+      if (visited[i]) continue;
+
+      const group = [data[i]];
+      visited[i] = true;
+
+      for (let j = i + 1; j < data.length; j++) {
+        if (visited[j]) continue;
+
+        const sim = this.cosineSimilarity(data[i].embedding, data[j].embedding);
+        if (sim > threshold) {
+          group.push(data[j]);
+          visited[j] = true;
+          consolidatedConcepts.push({
+            concept1: data[i].text,
+            concept2: data[j].text,
+            similarity: sim,
+          });
+        }
+      }
+
+      // Consolidate the group (e.g., just take the first, or merge)
+      finalConsolidatedConcepts.push(group[0]); // Or you can customize how to merge
+    }
+
+    return {
+      finalConsolidatedConcepts: finalConsolidatedConcepts,
+      consolidatedConcepts: consolidatedConcepts,
+    };
+  }
+}

package/src/embedding_generation/consolidation/write_consolidated_data.ts

@@ -1,98 +1,98 @@
-import qdrantClient from "../../services/qdrant_service";
-
-export class WriteConsolidatedData {
-  async writeConsolidatedData(
-    data: {
-      source_taxonomy: {
-        id: string;
-        text: string;
-        type: string;
-        reference: string;
-      }[];
-      global_updates: {
-        id: string;
-        currentPayload: {
-          _sources: string[];
-          text: string;
-        };
-        sourceIdToAdd: string;
-      }[];
-      global_inserts: {
-        id: string;
-        vector: number[];
-        payload: {
-          _sources: string[];
-          text: string;
-        };
-      }[];
-    },
-    generation_requests: any[],
-    source_id: string
-  ) {
-    if (data.global_updates.length > 0) {
-      await this.writeGlobalUpdates(data.global_updates);
-    }
-    if (data.global_inserts.length > 0) {
-      await this.writeGlobalInserts(data.global_inserts);
-    }
-
-    return {
-      status: "success",
-    };
-  }
-
-  async writeGlobalUpdates(
-    data: {
-      id: string;
-      currentPayload: {
-        _sources: string[];
-        text: string;
-      };
-      sourceIdToAdd: string;
-    }[]
-  ) {
-    const operations = data.map((e) => {
-      return {
-        set_payload: {
-          points: [e.id],
-          payload: {
-            text: e.currentPayload.text,
-            _sources: [e.sourceIdToAdd, ...e.currentPayload._sources],
-          },
-        },
-      };
-    });
-    await qdrantClient.batchUpdate("concepts-vectors", {
-      wait: true,
-      operations: operations,
-    });
-  }
-
-  async writeGlobalInserts(
-    data: {
-      id: string;
-      vector: number[];
-      payload: {
-        _sources: string[];
-        text: string;
-      };
-    }[]
-  ) {
-    await qdrantClient.upsert("concepts-vectors", {
-      wait: true,
-      points: data,
-    });
-  }
-
-  async wiriteToMondo(
-    source_taxonomy: any[],
-    generation_requests: any[],
-    source_id: string
-  ) {
-    const mondo_data = source_taxonomy.map((e) => {
-      return {
-        id: e.id,
-      };
-    });
-  }
-}
+import qdrantClient from "../../services/qdrant_service";
+
+export class WriteConsolidatedData {
+  async writeConsolidatedData(
+    data: {
+      source_taxonomy: {
+        id: string;
+        text: string;
+        type: string;
+        reference: string;
+      }[];
+      global_updates: {
+        id: string;
+        currentPayload: {
+          _sources: string[];
+          text: string;
+        };
+        sourceIdToAdd: string;
+      }[];
+      global_inserts: {
+        id: string;
+        vector: number[];
+        payload: {
+          _sources: string[];
+          text: string;
+        };
+      }[];
+    },
+    generation_requests: any[],
+    source_id: string
+  ) {
+    if (data.global_updates.length > 0) {
+      await this.writeGlobalUpdates(data.global_updates);
+    }
+    if (data.global_inserts.length > 0) {
+      await this.writeGlobalInserts(data.global_inserts);
+    }
+
+    return {
+      status: "success",
+    };
+  }
+
+  async writeGlobalUpdates(
+    data: {
+      id: string;
+      currentPayload: {
+        _sources: string[];
+        text: string;
+      };
+      sourceIdToAdd: string;
+    }[]
+  ) {
+    const operations = data.map((e) => {
+      return {
+        set_payload: {
+          points: [e.id],
+          payload: {
+            text: e.currentPayload.text,
+            _sources: [e.sourceIdToAdd, ...e.currentPayload._sources],
+          },
+        },
+      };
+    });
+    await qdrantClient.batchUpdate("concepts-vectors", {
+      wait: true,
+      operations: operations,
+    });
+  }
+
+  async writeGlobalInserts(
+    data: {
+      id: string;
+      vector: number[];
+      payload: {
+        _sources: string[];
+        text: string;
+      };
+    }[]
+  ) {
+    await qdrantClient.upsert("concepts-vectors", {
+      wait: true,
+      points: data,
+    });
+  }
+
+  async wiriteToMondo(
+    source_taxonomy: any[],
+    generation_requests: any[],
+    source_id: string
+  ) {
+    const mondo_data = source_taxonomy.map((e) => {
+      return {
+        id: e.id,
+      };
+    });
+  }
+}

package/src/embedding_generation/generate_embeddings.ts

@@ -1,42 +1,42 @@
-import { OpenAiService } from "../services/open_ai_service";
-
-export class GenerateEmbeddings {
-  constructor(private readonly openAiService: OpenAiService) {}
-
-  async generateEmbeddings(
-    concepts_facts: { text: string; type: string; reference: string }[]
-  ) {
-    const texts = concepts_facts.map((e: any) => e.text);
-    const response = await this.openAiService.sendEmbeddingRequest(texts);
-    if (response.status_code !== 200) {
-      console.log(response);
-      throw new Error(response.message);
-    }
-    const embeddings = response.data.data.sort((a: any) => a.index);
-    const model = response.data.model;
-    const usage = response.data.usage;
-    const embeddings_map = [];
-    for (let i = 0; i < concepts_facts.length; i++) {
-      embeddings_map.push({
-        text: concepts_facts[i].text,
-        type: concepts_facts[i].type,
-        embedding: embeddings[i].embedding,
-        reference: concepts_facts[i].reference,
-      });
-    }
-    // return embeddings_map;
-    return {
-      concepts_facts: embeddings_map,
-      metadata: {
-        req_time: new Date().toISOString(),
-        req_type: {
-          type: "embedding",
-        },
-        req_tokens: usage.prompt_tokens,
-        res_tokens: usage.completion_tokens,
-        model: model,
-        usage: usage,
-      },
-    };
-  }
-}
+import { OpenAiService } from "../services/open_ai_service";
+
+export class GenerateEmbeddings {
+  constructor(private readonly openAiService: OpenAiService) {}
+
+  async generateEmbeddings(
+    concepts_facts: { text: string; type: string; reference: string }[]
+  ) {
+    const texts = concepts_facts.map((e: any) => e.text);
+    const response = await this.openAiService.sendEmbeddingRequest(texts);
+    if (response.status_code !== 200) {
+      console.log(response);
+      throw new Error(response.message);
+    }
+    const embeddings = response.data.data.sort((a: any) => a.index);
+    const model = response.data.model;
+    const usage = response.data.usage;
+    const embeddings_map = [];
+    for (let i = 0; i < concepts_facts.length; i++) {
+      embeddings_map.push({
+        text: concepts_facts[i].text,
+        type: concepts_facts[i].type,
+        embedding: embeddings[i].embedding,
+        reference: concepts_facts[i].reference,
+      });
+    }
+    // return embeddings_map;
+    return {
+      concepts_facts: embeddings_map,
+      metadata: {
+        req_time: new Date().toISOString(),
+        req_type: {
+          type: "embedding",
+        },
+        req_tokens: usage.prompt_tokens,
+        res_tokens: usage.completion_tokens,
+        model: model,
+        usage: usage,
+      },
+    };
+  }
+}

package/src/embedding_generation/parse_embedding_response.ts

@@ -1,31 +1,31 @@
-import { LocalConsolidation } from "./consolidation/local_consolidation";
-
-export class ParseEmbeddingResponse {
-  parse(
-    response: any,
-    concepts_facts: { text: string; type: string; reference: string }[]
-  ) {
-    const data = response.data;
-    const model = data.model;
-    const usage = data.usage;
-    const rawEmbeddings = data.data;
-    const sorted = rawEmbeddings.sort((e: any) => e.index);
-    const embeddings = sorted.map((e: any) => e.embedding);
-    const embeddings_map = new Map();
-    const embeddings_map_array = [];
-    for (let i = 0; i < concepts_facts.length; i++) {
-      embeddings_map_array.push({
-        text: concepts_facts[i].text,
-        type: concepts_facts[i].type,
-        embedding: embeddings[i],
-        reference: concepts_facts[i].reference,
-      });
-    }
-    const embeddingMap = embeddings_map;
-    const consolidated = new LocalConsolidation().consolidate(
-      embeddings_map_array,
-      "sourceID"
-    );
-    return consolidated;
-  }
-}
+import { LocalConsolidation } from "./consolidation/local_consolidation";
+
+export class ParseEmbeddingResponse {
+  parse(
+    response: any,
+    concepts_facts: { text: string; type: string; reference: string }[]
+  ) {
+    const data = response.data;
+    const model = data.model;
+    const usage = data.usage;
+    const rawEmbeddings = data.data;
+    const sorted = rawEmbeddings.sort((e: any) => e.index);
+    const embeddings = sorted.map((e: any) => e.embedding);
+    const embeddings_map = new Map();
+    const embeddings_map_array = [];
+    for (let i = 0; i < concepts_facts.length; i++) {
+      embeddings_map_array.push({
+        text: concepts_facts[i].text,
+        type: concepts_facts[i].type,
+        embedding: embeddings[i],
+        reference: concepts_facts[i].reference,
+      });
+    }
+    const embeddingMap = embeddings_map;
+    const consolidated = new LocalConsolidation().consolidate(
+      embeddings_map_array,
+      "sourceID"
+    );
+    return consolidated;
+  }
+}

package/src/enums/card_type_enum.ts

@@ -1,6 +1,6 @@
-export enum CardTypeEnum {
-  FLASH = "flash",
-  CLOZE = "cloze",
-  MATCH = "match",
-  MCQ = "mcq",
-}
+export enum CardTypeEnum {
+  FLASH = "flash",
+  CLOZE = "cloze",
+  MATCH = "match",
+  MCQ = "mcq",
+}