only_ever_generator 0.9.5 → 0.9.7

package/dist/embedding_generation/consolidation/global_consolidation.js

@@ -0,0 +1,75 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GlobalConsolidation = void 0;
+const qdrant_db_methods_1 = require("../../helper/qdrant_db_methods");
+class GlobalConsolidation {
+    consolidate(locally_consolidated_concepts_facts, sourceId, threshold) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            // / for all concepts_facts, find the ,most similar concepts_facts in qdrant, threshold of 0.8
+            const similarConcepts = yield (0, qdrant_db_methods_1.getSimilarConcepts)("concepts-vectors", locally_consolidated_concepts_facts, threshold !== null && threshold !== void 0 ? threshold : 0.8);
+            const taxonomyConcepts = [];
+            const globalUpdatesOps = [];
+            const globalInsertsOps = [];
+            for (const index in similarConcepts) {
+                const points = similarConcepts[index].points;
+                const originalConcept = locally_consolidated_concepts_facts[index];
+                if (points.length == 0) {
+                    taxonomyConcepts.push({
+                        id: originalConcept.id,
+                        text: originalConcept.text,
+                        type: originalConcept.type,
+                        reference: originalConcept.reference,
+                    });
+                    globalInsertsOps.push({
+                        id: originalConcept.id,
+                        vector: originalConcept.embedding,
+                        payload: {
+                            _sources: [sourceId],
+                            text: originalConcept.text,
+                        },
+                    });
+                }
+                else {
+                    const consolidatedId = points[0].id;
+                    const currentPayload = points[0].payload;
+                    // const currentSources = currentPayload?['_sources'] ?? [];
+                    taxonomyConcepts.push({
+                        id: consolidatedId.toString(),
+                        text: originalConcept.text,
+                        type: originalConcept.type,
+                        reference: originalConcept.reference,
+                    });
+                    globalUpdatesOps.push({
+                        id: consolidatedId.toString(),
+                        sourceIdToAdd: sourceId,
+                        currentPayload: {
+                            // _sources: currentPayload['_sources'] ?? [],
+                            _sources: ((_a = currentPayload === null || currentPayload === void 0 ? void 0 : currentPayload._sources) !== null && _a !== void 0 ? _a : []),
+                            text: ((_b = currentPayload === null || currentPayload === void 0 ? void 0 : currentPayload.text) !== null && _b !== void 0 ? _b : "").toString(),
+                        },
+                    });
+                }
+            }
+            return {
+                source_taxonomy: taxonomyConcepts,
+                global_updates: globalUpdatesOps,
+                global_inserts: globalInsertsOps,
+            };
+            /// if threshold is below 0.8, then add the concept_fact to qdrant
+            /// if threshold is above 0.8, then replace the id of the local concept_fact with the id of the global concept_fact
+            /// add _source.id to qdrant
+            /// return the concepts_facts
+        });
+    }
+}
+exports.GlobalConsolidation = GlobalConsolidation;

package/dist/embedding_generation/consolidation/local_consolidation.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalConsolidation = void 0;
+const crypto_1 = require("crypto");
+class LocalConsolidation {
+    constructor() {
+        this.cosineSimilarity = (a, b) => {
+            const dotProduct = a.reduce((acc, val, index) => acc + val * b[index], 0);
+            const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
+            const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
+            return dotProduct / (magnitudeA * magnitudeB);
+        };
+    }
+    consolidate(concepts_facts, sourceId) {
+        try {
+            const sourceIds = [];
+            const sourceTaxonomyOps = [];
+            const globalConceptOps = [];
+            const consolidatedConcepts = [];
+            const concepts = concepts_facts.map((e) => {
+                return {
+                    id: (0, crypto_1.randomUUID)().toString(),
+                    text: e.text,
+                    reference: e.reference,
+                    embedding: e.embedding,
+                    type: e.type,
+                };
+            });
+            const afterConsolidation = this.consolidateSimilarEmbeddings(concepts);
+            sourceIds.push(JSON.stringify(sourceId));
+            console.log(afterConsolidation);
+            for (var concept of afterConsolidation.finalConsolidatedConcepts) {
+                sourceTaxonomyOps.push({
+                    text: concept.text,
+                    id: concept.id,
+                    embedding: concept.embedding,
+                    type: concept.type,
+                    reference: concept.reference,
+                });
+                globalConceptOps.push({
+                    id: concept.id.toString(),
+                    vector: concept.embedding,
+                    payload: {
+                        _sources: [sourceId],
+                        text: concept.text,
+                    },
+                });
+            }
+            consolidatedConcepts.push(...afterConsolidation.consolidatedConcepts);
+            return {
+                globalConceptOps: globalConceptOps,
+                sourceIds: sourceIds,
+                consolidatedConcepts: consolidatedConcepts,
+                sourceTaxonomyOps: sourceTaxonomyOps,
+            };
+        }
+        catch (e) {
+            console.error("Error occurred while converting the parsed array to db operations:", e);
+            throw e;
+        }
+        // const concepts = concepts_facts.map((e) => {
+        //   return {
+        //     id: randomUUID().toString(),
+        //     text: e.text,
+        //     type: e.type,
+        //     embedding: e.embedding,
+        //   };
+        // });
+        // const consolidated = this._consolidateSimilarEmbeddings(concepts);
+        // return consolidated;
+    }
+    consolidateSimilarEmbeddings(data, threshold = 0.65) {
+        const finalConsolidatedConcepts = [];
+        const visited = new Array(data.length).fill(false);
+        const consolidatedConcepts = [];
+        for (let i = 0; i < data.length; i++) {
+            if (visited[i])
+                continue;
+            const group = [data[i]];
+            visited[i] = true;
+            for (let j = i + 1; j < data.length; j++) {
+                if (visited[j])
+                    continue;
+                const sim = this.cosineSimilarity(data[i].embedding, data[j].embedding);
+                if (sim > threshold) {
+                    group.push(data[j]);
+                    visited[j] = true;
+                    consolidatedConcepts.push({
+                        concept1: data[i].text,
+                        concept2: data[j].text,
+                        similarity: sim,
+                    });
+                }
+            }
+            // Consolidate the group (e.g., just take the first, or merge)
+            finalConsolidatedConcepts.push(group[0]); // Or you can customize how to merge
+        }
+        return {
+            finalConsolidatedConcepts: finalConsolidatedConcepts,
+            consolidatedConcepts: consolidatedConcepts,
+        };
+    }
+}
+exports.LocalConsolidation = LocalConsolidation;

package/dist/embedding_generation/consolidation/write_consolidated_data.js

@@ -0,0 +1,68 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WriteConsolidatedData = void 0;
+const qdrant_service_1 = __importDefault(require("../../services/qdrant_service"));
+class WriteConsolidatedData {
+    writeConsolidatedData(data, generation_requests, source_id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (data.global_updates.length > 0) {
+                yield this.writeGlobalUpdates(data.global_updates);
+            }
+            if (data.global_inserts.length > 0) {
+                yield this.writeGlobalInserts(data.global_inserts);
+            }
+            return {
+                status: "success",
+            };
+        });
+    }
+    writeGlobalUpdates(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const operations = data.map((e) => {
+                return {
+                    set_payload: {
+                        points: [e.id],
+                        payload: {
+                            text: e.currentPayload.text,
+                            _sources: [e.sourceIdToAdd, ...e.currentPayload._sources],
+                        },
+                    },
+                };
+            });
+            yield qdrant_service_1.default.batchUpdate("concepts-vectors", {
+                wait: true,
+                operations: operations,
+            });
+        });
+    }
+    writeGlobalInserts(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield qdrant_service_1.default.upsert("concepts-vectors", {
+                wait: true,
+                points: data,
+            });
+        });
+    }
+    wiriteToMondo(source_taxonomy, generation_requests, source_id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const mondo_data = source_taxonomy.map((e) => {
+                return {
+                    id: e.id,
+                };
+            });
+        });
+    }
+}
+exports.WriteConsolidatedData = WriteConsolidatedData;

package/dist/embedding_generation/generate_embeddings.js

@@ -0,0 +1,53 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GenerateEmbeddings = void 0;
+class GenerateEmbeddings {
+    constructor(openAiService) {
+        this.openAiService = openAiService;
+    }
+    generateEmbeddings(concepts_facts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const texts = concepts_facts.map((e) => e.text);
+            const response = yield this.openAiService.sendEmbeddingRequest(texts);
+            if (response.status_code !== 200) {
+                throw new Error(response.message);
+            }
+            const embeddings = response.data.data.sort((a) => a.index);
+            const model = response.data.model;
+            const usage = response.data.usage;
+            const embeddings_map = [];
+            for (let i = 0; i < concepts_facts.length; i++) {
+                embeddings_map.push({
+                    text: concepts_facts[i].text,
+                    type: concepts_facts[i].type,
+                    embedding: embeddings[i].embedding,
+                    reference: concepts_facts[i].reference,
+                });
+            }
+            // return embeddings_map;
+            return {
+                concepts_facts: embeddings_map,
+                metadata: {
+                    req_time: new Date().toISOString(),
+                    req_type: {
+                        type: "embedding",
+                    },
+                    req_tokens: usage.prompt_tokens,
+                    res_tokens: usage.completion_tokens,
+                    model: model,
+                    usage: usage,
+                },
+            };
+        });
+    }
+}
+exports.GenerateEmbeddings = GenerateEmbeddings;

package/dist/embedding_generation/parse_embedding_response.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParseEmbeddingResponse = void 0;
+const local_consolidation_1 = require("./consolidation/local_consolidation");
+class ParseEmbeddingResponse {
+    parse(response, concepts_facts) {
+        const data = response.data;
+        const model = data.model;
+        const usage = data.usage;
+        const rawEmbeddings = data.data;
+        const sorted = rawEmbeddings.sort((e) => e.index);
+        const embeddings = sorted.map((e) => e.embedding);
+        const embeddings_map = new Map();
+        const embeddings_map_array = [];
+        for (let i = 0; i < concepts_facts.length; i++) {
+            embeddings_map_array.push({
+                text: concepts_facts[i].text,
+                type: concepts_facts[i].type,
+                embedding: embeddings[i],
+                reference: concepts_facts[i].reference,
+            });
+        }
+        const embeddingMap = embeddings_map;
+        const consolidated = new local_consolidation_1.LocalConsolidation().consolidate(embeddings_map_array, "sourceID");
+        return consolidated;
+    }
+}
+exports.ParseEmbeddingResponse = ParseEmbeddingResponse;

package/dist/gap_fill/calculate_gap_fill.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gapFilling = gapFilling;
+function isEmpty(obj) {
+    return Object.keys(obj).length === 0;
+}
+function gapFilling(typologyResponse, cardgenResponse) {
+    var _a, _b, _c, _d, _e;
+    let allConcepts = [];
+    let allFacts = [];
+    let generatedConceptsList = [];
+    let generatedFactsList = [];
+    let remainingConcepts = [];
+    let remainingFacts = [];
+    if (!isEmpty(typologyResponse)) {
+        allConcepts.push(...((_a = typologyResponse.concepts) !== null && _a !== void 0 ? _a : []));
+        allFacts.push(...((_b = typologyResponse === null || typologyResponse === void 0 ? void 0 : typologyResponse.facts) !== null && _b !== void 0 ? _b : []));
+    }
+    if (!isEmpty(cardgenResponse)) {
+        allConcepts.push(...((_c = cardgenResponse.missing_concepts) !== null && _c !== void 0 ? _c : []));
+        allFacts.push(...((_d = cardgenResponse.missing_facts) !== null && _d !== void 0 ? _d : []));
+    }
+    if (cardgenResponse.cards_data !== undefined &&
+        ((_e = cardgenResponse.cards_data) === null || _e === void 0 ? void 0 : _e.length) != 0) {
+        for (let card of cardgenResponse.cards_data) {
+            if (card.concepts.length != 0) {
+                generatedConceptsList.push(...card.concepts);
+            }
+            if (card.facts.length != 0) {
+                generatedFactsList.push(...card.facts);
+            }
+        }
+        let generatedConceptsSet = Array.from(new Set(generatedConceptsList));
+        let generatedFactsSet = Array.from(new Set(generatedFactsList));
+        remainingConcepts = allConcepts.filter((item) => !generatedConceptsSet.includes(item));
+        remainingFacts = allFacts.filter((item) => !generatedFactsSet.includes(item));
+    }
+    return {
+        remainingConcepts: remainingConcepts,
+        remainingFacts: remainingFacts,
+    };
+}

package/dist/helper/qdrant_db_methods.js

@@ -0,0 +1,62 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSimilarConcepts = exports.addEmbeddingsToCollection = exports.getCollection = exports.createCollection = void 0;
+const qdrant_service_1 = __importDefault(require("../services/qdrant_service"));
+const createCollection = (collectionName) => __awaiter(void 0, void 0, void 0, function* () {
+    yield qdrant_service_1.default.createCollection(collectionName, {
+        vectors: {
+            size: 1536,
+            distance: "Cosine",
+        },
+    });
+});
+exports.createCollection = createCollection;
+const getCollection = () => __awaiter(void 0, void 0, void 0, function* () {
+    const collection = yield qdrant_service_1.default.getCollections();
+    return collection;
+});
+exports.getCollection = getCollection;
+const getCorrespondingConcepts = (collectionName, embeddings, threshold) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const searchQuery = embeddings.map((e) => {
+            return {
+                query: e.embedding,
+                limit: 1,
+                score_threshold: threshold,
+                with_payload: true,
+            };
+        });
+        const results = yield qdrant_service_1.default.queryBatch(collectionName, {
+            searches: searchQuery,
+        });
+        return results;
+    }
+    catch (error) {
+        console.log(error);
+        throw error;
+    }
+});
+exports.getSimilarConcepts = getCorrespondingConcepts;
+const addEmbeddingsToCollection = (collectionName, embeddings) => __awaiter(void 0, void 0, void 0, function* () {
+    const CHUNK_SIZE = 1000;
+    for (let i = 0; i < embeddings.length; i += CHUNK_SIZE) {
+        const batch = embeddings.slice(i, i + CHUNK_SIZE);
+        yield qdrant_service_1.default.upsert(collectionName, {
+            wait: true,
+            points: batch,
+        });
+    }
+});
+exports.addEmbeddingsToCollection = addEmbeddingsToCollection;

package/dist/index.js

@@ -0,0 +1,96 @@
+"use strict";
+// import express from "express";
+// import {
+//   returnCardResponse,
+//   returnHeadings,
+//   returnSourceData,
+// } from "./constants/source_data";
+// import config from "./config";
+// const app = express();
+// const port = 3000;
+// import { returnPromptData } from "./constants/prompt_data";
+// import { GenerateCards } from "./card_gen/generate_cards";
+// import { OpenAiService } from "./services/open_ai_service";
+// import { ParseCardResponse } from "./parse/parse_card_response";
+// import { returnTypologyPrompt } from "./constants/prompts/typology_prompt";
+// import { returnCardGenPrompt } from "./constants/prompts/card_gen_prompt";
+// import { GenerateArgs } from "./utils/generate_args";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OnlyEverGenerator = void 0;
+const app_1 = require("./bootstrap/app");
+Object.defineProperty(exports, "OnlyEverGenerator", { enumerable: true, get: function () { return app_1.OnlyEverGenerator; } });
+// . All the Codes Below uses express and are strictly for development purpose, while publishing the package, comment everything
+// below this line
+// let oeGen = new OnlyEverGenerator(config.openAIKey, "gpt-4o-mini", {
+//   prompt: returnPromptData(),
+//   content: returnSourceData(),
+// });
+// app.get("/", async (req, res) => {
+//   // let data = oeGen.returnParsedContent();
+//   // let parsedData = parseResponse()
+//   let cardPrompt = "";
+//   res.send(cardPrompt);
+// });
+// app.get("/openAI", async (req, res) => {
+//   // let prompt = returnPromt();
+//   // let prompt = returnCardGenPrompt();
+//   //   let content = returnSourceData().toString()
+//   //   let headings = returnHeadings();
+//   //   // let aiRequest = await openAIRequest(prompt,content);
+//   let aiRequest = await oeGen.generate(true, true);
+//   res.send(aiRequest);
+// });
+// app.get("/parseCardData", async (req, res) => {
+//   let cardResp = returnCardResponse();
+//   let headings = returnHeadings();
+//   cardResp.metadata = {
+//     req_time: cardResp.generated_at ?? new Date(),
+//     req_type: "card",
+//     req_tokens: cardResp.usage_data?.prompt_tokens,
+//     res_tokens: cardResp.usage_data?.completion_tokens,
+//     model: "40-mini",
+//   };
+//   let parsedData = new ParseCardResponse().parse(cardResp, false, {});
+//   res.send(parsedData);
+// });
+// app.get("/generate-cards", async (req, res) => {
+//   let cardResp = embeddingsResp;
+//   oeGen.parsedContent.taxonomy = cardResp;
+//   let parsedData = await oeGen.generateCard(
+//     returnCardGenPrompt(),
+//     JSON.stringify(cardResp),
+//     false
+//   );
+//   res.send(parsedData);
+// });
+// app.get("/typology", async (req, res) => {
+//   {
+//     let typologyRequest = await oeGen.generate(true, false);
+//     res.send(typologyRequest);
+//   }
+// });
+// app.get("/local-consolidation", async (req, res) => {
+//   const concepts_facts = embeddingsResp.concepts_facts;
+//   let locallyConsolidated = new LocalConsolidation().consolidate(
+//     concepts_facts,
+//     "sourceID"
+//   );
+//   res.send(locallyConsolidated);
+// });
+// app.get("/global-consolidation", async (req, res) => {
+//   const concepts_facts = embeddingsResp.concepts_facts;
+//   // let globallyConsolidated = await new GlobalConsolidation().consolidate(
+//   //   concepts_facts,
+//   //   "sourceID",
+//   //   0.2
+//   // );
+//   // let globallyConsolidated = await oeGen.globalConsolidation(
+//   //   concepts_facts,
+//   //   "sourceID",
+//   //   0.2
+//   // );
+//   // res.send(globallyConsolidated);
+// });
+// app.listen(port, () => {
+//   console.log(`Example app listening at http://localhost:${port}`);
+// });

package/dist/logger.js

@@ -0,0 +1,41 @@
+"use strict";
+/// this method will call out atlas function and will write to a doc, incase of any errors:
+/// this is only for developmemt use
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ErrorLogger = void 0;
+const axios_1 = __importDefault(require("axios"));
+class ErrorLogger {
+    constructor(data) {
+        this.data = data;
+    }
+    log() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                let response = yield axios_1.default.post("https://us-east-1.aws.data.mongodb-api.com/app/oe-phase1-tkmsy/endpoint/oe_gen_logger", {
+                    data: this.data,
+                }, {
+                    headers: {
+                        "Content-Type": ["application/json"],
+                    },
+                });
+                return response;
+            }
+            catch (e) {
+                console.log(e);
+            }
+        });
+    }
+}
+exports.ErrorLogger = ErrorLogger;