only_ever_generator 0.0.1

package/dist/parse_response/response_format_card.js

@@ -0,0 +1,210 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseResponse = exports.returnResponse = void 0;
+const responseData = {
+    "flash_cards": [
+        {
+            "question": "What is the primary function of the 'Electrolysis'?",
+            "answer": "Electrolysis is the passing of a direct electric current through an electrolyte producing chemical reactions at the electrodes and decomposition of the materials.",
+            "heading": "Overview"
+        },
+        {
+            "question": "What did Michael Faraday discover while studying the process of electrolysis under Humphry Davy?",
+            "answer": "Michael Faraday discovered two laws of electrolysis.",
+            "heading": "History"
+        },
+        {
+            "question": "What can electrolysis be used for in the manufacturing sector?",
+            "answer": "Electrolysis can be used for electroplating and electrochemical machining.",
+            "heading": "Manufacturing processes"
+        },
+        {
+            "question": "In the context of electrolysis, what does the term 'Decomposition potential' refer to?",
+            "answer": "Decomposition potential or decomposition voltage refers to the minimum voltage between anode and cathode of an electrolytic cell needed for electrolysis to occur.",
+            "heading": "Decomposition potential"
+        }
+    ],
+    "mcqs": [
+        {
+            "question": "What is the primary purpose of 'Electrometallurgy'?",
+            "answers": [
+                {
+                    "answer": "Production of aluminium",
+                    "is_correct": true
+                },
+                {
+                    "answer": "Chlorine production",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Purifying copper",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Rust removal",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Hydrogen production",
+                    "is_correct": false
+                }
+            ],
+            "heading": "Industrial uses"
+        },
+        {
+            "question": "What can electrolysis be used for in the context of batteries?",
+            "answers": [
+                {
+                    "answer": "Spontaneous redox reactions",
+                    "is_correct": true
+                },
+                {
+                    "answer": "Energy-releasing reactions",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Disinfectant production",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Fuel production",
+                    "is_correct": false
+                },
+                {
+                    "answer": "Gas diffusion in reactors",
+                    "is_correct": false
+                }
+            ],
+            "heading": "Related processes"
+        }
+    ],
+    "cloze_cards": [
+        {
+            "question": "Electrolysis is the passing of a {{c0:direct electric current}} through an {{c1:electrolyte}} producing {{c2:chemical reactions}} at the {{c3:electrodes}} and {{c4:decomposition}} of the materials.",
+            "options": [
+                {
+                    "option": "direct electric current",
+                    "cloze": "c0"
+                },
+                {
+                    "option": "electrolyte",
+                    "cloze": "c1"
+                },
+                {
+                    "option": "chemical reactions",
+                    "cloze": "c2"
+                },
+                {
+                    "option": "electrodes",
+                    "cloze": "c3"
+                },
+                {
+                    "option": "decomposition",
+                    "cloze": "c4"
+                },
+                {
+                    "option": "metallic objects",
+                    "cloze": null
+                }
+            ]
+        },
+        {
+            "question": "In electrolysis, the decomposition potential or decomposition voltage refers to the minimum voltage between {{c0:anode}} and {{c1:cathode}} of an electrolytic cell needed for electrolysis to occur.",
+            "options": [
+                {
+                    "option": "anode",
+                    "cloze": "c0"
+                },
+                {
+                    "option": "cathode",
+                    "cloze": "c1"
+                },
+                {
+                    "option": "electrolyte",
+                    "cloze": null
+                },
+                {
+                    "option": "chemical reactions",
+                    "cloze": null
+                },
+                {
+                    "option": "oxygen",
+                    "cloze": null
+                },
+                {
+                    "option": "hydrogen",
+                    "cloze": null
+                }
+            ]
+        }
+    ]
+};
+function returnResponse() {
+    return responseData;
+}
+exports.returnResponse = returnResponse;
+function parseResponse() {
+    let cardData = [];
+    let data = returnResponse();
+    let keys = Object.keys(data);
+    if (keys) {
+        for (let elem of keys) {
+            if (elem == 'flash_cards') {
+                cardData.push(...parseFlashCard(data.flash_cards));
+            }
+            else if (elem == 'cloze_cards') {
+                cardData.push(...parseClozeCards(data.cloze_cards));
+            }
+            else if (elem == 'mcqs') {
+                cardData.push(...parseMcq(data.mcqs));
+            }
+        }
+    }
+    return cardData;
+}
+exports.parseResponse = parseResponse;
+/// takes array of 
+function parseFlashCard(cards) {
+    let flashCardData = [];
+    for (let elem of cards) {
+        flashCardData.push({
+            "type": "flash",
+            "heading": elem.heading,
+            "content": JSON.stringify({
+                "front_content": elem.question,
+                "back_content": elem.answer
+            })
+        });
+    }
+    return flashCardData;
+}
+/// takes array of 
+function parseMcq(cardsData) {
+    let mcqCards = [];
+    for (let elem of cardsData) {
+        mcqCards.push({
+            "type": "mcq",
+            "heading": elem.heading,
+            "content": JSON.stringify({
+                "question": elem.question,
+                "answers": elem.answers
+            })
+        });
+    }
+    return mcqCards;
+}
+/// takes array of 
+function parseClozeCards(cardsData) {
+    let clozeCards = [];
+    for (let elem of cardsData) {
+        clozeCards.push({
+            "type": "cloze",
+            "heading": elem.heading,
+            "content": JSON.stringify({
+                "question": elem.question,
+                "options": elem.options
+            })
+        });
+    }
+    return clozeCards;
+}

package/dist/parse_response/response_format_typology.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.returnTypologyData = void 0;
+const typologyResponse = {
+    "usage_data": {
+        "prompt_tokens": 11611,
+        "completion_tokens": 441,
+        "total_tokens": 12052
+    },
+    "generated_content": {
+        "field": [
+            "Sciences",
+            "Technology & Engineering",
+            "Education, Learning & Personal Development"
+        ],
+        "concepts": [
+            "Electrolysis",
+            "Faraday's Laws of Electrolysis",
+            "Electrolytic Cell",
+            "Decomposition Potential",
+            "Oxidation and Reduction at the Electrodes",
+            "Electrolysis of Water",
+            "Electrolysis of Carbon Dioxide",
+            "Electrocrystallization"
+        ],
+        "facts": [
+            "Electrolysis is the passing of a direct electric current through an electrolyte producing chemical reactions at the electrodes and decomposition of the materials.",
+            "In electrolysis, the quantity of the products is proportional to the current, and when two or more electrolytic cells are connected in series to the same power source, the products produced in the cells are proportional to their equivalent weight.",
+            "The main components required to achieve electrolysis are an electrolyte, electrodes, and an external power source.",
+            "Faraday's laws of electrolysis detail the amount of the products of electrolysis is related to the number of electrons in the reaction at the electrodes.",
+            "Decomposition potential or decomposition voltage refers to the minimum voltage between anode and cathode of an electrolytic cell that is needed for electrolysis to occur.",
+            "The electrochemical reduction of carbon dioxide can produce value-added chemicals such as methane, ethylene, and ethanol."
+        ],
+        "generate_cards": true,
+        "summary_cards": [
+            "Electrolysis is the process of passing direct electric current through an electrolyte, resulting in chemical reactions and the decomposition of materials.",
+            "Faraday's laws of electrolysis determine the relationship between the amounts of products generated and the electrons involved in the reaction at the electrodes.",
+            "Decomposition potential is the minimum voltage required between anode and cathode for electrolysis to occur.",
+            "The electrochemical reduction of carbon dioxide is a potential method for producing valuable chemicals such as methane, ethylene, and ethanol."
+        ]
+    },
+    "generated_at": "Tue, 20 Jan 1970 21:17:46 GMT"
+};
+function returnTypologyData() {
+    return typologyResponse;
+}
+exports.returnTypologyData = returnTypologyData;

package/dist/service/open_ai_request.js

@@ -0,0 +1,58 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openAIRequest = void 0;
+const axios_1 = __importDefault(require("axios"));
+const parse_openai_response_js_1 = require("../utils/parse_openai_response.js");
+function openAIRequest(content, prompt, token, model) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            let message = [
+                {
+                    "role": "system", "content": prompt
+                }, {
+                    "role": "user", "content": content
+                }
+            ];
+            const url = 'https://api.openai.com/v1/chat/completions';
+            let response = yield axios_1.default.post(url, 
+            //data
+            {
+                "model": model,
+                "messages": message,
+                "response_format": { "type": "json_object" },
+            }, 
+            ///config
+            {
+                headers: {
+                    Authorization: "Bearer " + token,
+                    "Content-Type": ['application/json']
+                },
+            });
+            if (response.status == 200) {
+                console.log('success');
+                return (0, parse_openai_response_js_1.parseOpenAiSuccessResponse)(response.data);
+                // return JSON.parse(response.data.choices[0].message.content);
+            }
+            else {
+                console.log('failed');
+                return response.statusText;
+            }
+        }
+        catch (err) {
+            return (0, parse_openai_response_js_1.parseOpenAiFailureResponse)(err.response);
+        }
+    });
+}
+exports.openAIRequest = openAIRequest;

package/dist/typology_gen/generate_typology.js

@@ -0,0 +1,66 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GenerateTypology = void 0;
+class GenerateTypology {
+    constructor(openAiService, prompt, content) {
+        this.prompt = '';
+        this.content = '';
+        this.openAiService = openAiService;
+        this.prompt = prompt;
+        this.content = content;
+    }
+    generate() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const response = yield ((_a = this.openAiService) === null || _a === void 0 ? void 0 : _a.sendRequest(this.prompt, this.content));
+            response['type'] = 'typology';
+            response.metadata = {
+                "req_time": response.created_at,
+                "req_type": response.type,
+                "req_tokens": response.usage_data.prompt_tokens,
+                "res_tokens": response.usage_data.completion_tokens,
+            };
+            if (response.status_code == 200) {
+                return this.parseTypologyOnSuccess(response);
+            }
+            else {
+                return response;
+            }
+        });
+    }
+    parseTypologyOnSuccess(responseData) {
+        return __awaiter(this, void 0, void 0, function* () {
+            responseData.metadata.status = "completed";
+            const generatedContent = responseData.generated_content;
+            return {
+                status_code: 200,
+                metadata: responseData.metadata,
+                field: generatedContent.field,
+                concepts: generatedContent.concepts,
+                facts: generatedContent.facts,
+                generate_cards: generatedContent.generate_cards,
+                summary_cards: generatedContent.summary_cards,
+                type: responseData.type,
+            };
+        });
+    }
+    parseTypologyOnFailure(responseData) {
+        return __awaiter(this, void 0, void 0, function* () {
+            responseData.metadata.status = 'failed';
+            return {
+                status_code: responseData.status_code,
+                metadata: responseData.metadata,
+            };
+        });
+    }
+}
+exports.GenerateTypology = GenerateTypology;

package/dist/utils/generate_args.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GenerateArgs = void 0;
+class GenerateArgs {
+    constructor(generate_card, generate_typology, generate_summary, prompts = {
+        typology_prompt: '',
+        card_gen_prompt: '',
+        summary_prompt: ''
+    }) {
+        this.generate_card = false;
+        this.generate_typology = false;
+        this.generate_summary = false;
+        this.prompts = {
+            typology_prompt: '',
+            card_gen_prompt: '',
+            summary_prompt: ''
+        };
+        this.generate_card = generate_card;
+        this.generate_typology = generate_typology;
+        this.generate_summary = generate_summary;
+        this.prompts = prompts;
+    }
+    getWhatNeedsToBeGenerated() {
+        let returnData = [];
+        if (this.generate_typology == true) {
+            returnData.push('generate_tyopology');
+        }
+        if (this.generate_summary == true) {
+            returnData.push('generate_summary');
+        }
+        if (this.generate_card == true) {
+            returnData.push('generate_card');
+        }
+        return returnData;
+    }
+}
+exports.GenerateArgs = GenerateArgs;

package/dist/utils/parse_openai_response.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseOpenAiFailureResponse = exports.parseOpenAiSuccessResponse = void 0;
+function parseOpenAiSuccessResponse(responseData) {
+    let choices = JSON.parse(responseData.choices[0].message.content);
+    let usuage = responseData.usage;
+    let createdTime = responseData.created;
+    return {
+        'status_code': 200,
+        'usage_data': usuage,
+        'generated_content': choices,
+        'generated_at': createdTime
+    };
+}
+exports.parseOpenAiSuccessResponse = parseOpenAiSuccessResponse;
+function parseOpenAiFailureResponse(errorResponse) {
+    let usuage = errorResponse.usuage;
+    // let statusCode = 
+    return {
+        'status_code': errorResponse.status,
+    };
+}
+exports.parseOpenAiFailureResponse = parseOpenAiFailureResponse;

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "only_ever_generator",
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "scripts": {
+    "start": "nodemon dist/index.js",
+    "build": "tsc",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "shree",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.14.2",
+    "nodemon": "^3.1.3",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  },
+  "dependencies": {
+    "axios": "^1.7.2",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2"
+  },
+  "eslintConfig": {
+    "parser": "typescript-eslint-parser",
+    "plugins": [
+      "typescript"
+    ]
+  }
+}

package/readme.md

@@ -0,0 +1,23 @@
+This is a NPM package that handles the all the ai generation for _source on OnlyEver.
+The Generation includes, Knowledge Taxonomy , Summary and Test Cards data for the _source.
+
+The class [OnlyEverGenerator] is the Entry point for the generation module it has two required fields, 
+[api_key] which is open ai token 
+[model] open ai model : 'gpt-3.5-turbo-1106'
+[content] source content : Array<BaseBlocks>
+
+The [OnlyEverGenerator] currently exposes a main method, [generate] which handles all types of generation and takes a Param 
+```
+generate(generate_typology = false,generate_card = false) method takes two args, generate_typology and generate_cards, which are both boolean.
+For Eg: 
+let oeGenerator = new OnlyEverGenerator(
+    api_key,
+    model,
+    [Block1,Block2,Block3], // oe content model
+);
+ 
+
+let responsesArray = await oeGenerator.generate(true,true);
+
+
+```

package/src/bootstrap/app.ts

@@ -0,0 +1,78 @@
+import { ParseSourceContent } from "../class/parse/parse_source_content";
+import { OpenAiService } from "../class/services/open_ai_service";
+import { returnCardGenPrompt } from "../constants/prompts/card_gen_prompt";
+import { returnTypologyPrompt } from "../constants/prompts/typology_prompt";
+import { GenerateTypology } from "../typology_gen/generate_typology";
+import { GenerateArgs } from "../utils/generate_args";
+
+export class OnlyEverGenerator{
+    public api_key: string = '';
+    public openAiService: OpenAiService;
+    parsedContent: string = '';
+    constructor(apiKey:string, model: string, content: Array<any>){
+        this.api_key = apiKey;
+        this.openAiService = new OpenAiService(apiKey,model ?? 'gpt-3.5-turbo-1106');
+        this.parsedContent = new ParseSourceContent(content).parse();
+    };
+
+    typologyResponse = {};
+    cardgenResponse = {};
+    summarizeResponse = {};
+
+    
+    
+
+    async generate( 
+       generate_card : boolean = false,
+       generate_typology: boolean =false,
+    ): Promise<Array<any>> {
+      let typologyPrompt = returnTypologyPrompt();
+      let cardPrompt = returnCardGenPrompt();
+      let args = new GenerateArgs(
+        generate_card,
+        generate_typology,
+        false,
+        {
+          typology_prompt: typologyPrompt,
+          card_gen_prompt: cardPrompt,
+          summary_prompt: ''
+        },
+        
+      );
+        const responseToReturn = [];
+        const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
+        for(let elem of whatNeedsToBeGenerated)
+        if(elem == 'generate_tyopology'){
+            this.typologyResponse = await this.generateTypology(args.prompts.typology_prompt ?? '');
+            responseToReturn.push(this.typologyResponse);
+        }else if(elem == 'generate_card'){
+
+            this.cardgenResponse =  await this.generateCard(args.prompts.card_gen_prompt ?? '', this.parsedContent +  JSON.stringify(this.typologyResponse));
+            responseToReturn.push(this.cardgenResponse);
+        }
+
+        return responseToReturn;
+        
+    }
+    
+    _returnParsedContent(){
+        return this.parsedContent;
+    }
+
+
+    async generateCard(prompt: string, content: string){
+        let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
+        response['type'] = 'card_gen';
+        return response;
+    }
+
+    async generateTypology(prompt:string){
+        let response =  await new GenerateTypology(
+            this.openAiService,
+            prompt,
+            this.parsedContent
+        ).generate();
+        return response;
+    }
+    
+}

package/src/class/parse/parse_source_content.ts

@@ -0,0 +1,71 @@
+export class ParseSourceContent{
+    public content: Array<any>;
+
+    titles_to_remove = ['See also', 'References', 'Further reading', 'External links', 'Notes and references', 'Bibliography', 'Notes', 'Cited sources'];
+
+    constructor(sourceContent:Array<any>){
+        this.content = sourceContent;
+    }
+
+    parse() {
+        let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(this.content);
+        let afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
+        return JSON.stringify(afterSanitized);    
+    }
+
+    removeSectionsByTitle(data: Array<any>){
+        let dataAfterRemoving = [];
+        for(let elem of data){
+            if(elem.block_type == 'heading' &&   this.titles_to_remove.includes(elem.content)){
+                continue;
+            }
+            if(elem.children){
+                elem.children = this.removeSectionsByTitle(elem.children)
+            }
+            dataAfterRemoving.push(elem)
+            
+        }
+        return dataAfterRemoving;
+    }
+    
+
+     sanitizeWikiContent(content: String) {
+        // Remove newline characters
+        content = content.replace(/\\n/g, ' ');
+    
+        // Remove internal link references, keeping only the link text
+        // Pattern explanation: [[link|text|index|wiki]] --> text
+        content = content.replace(/\[\[.*?\|(.*?)\|.*?\|wiki\]\]/g, '$1');
+    
+        // Remove external links, keeping only the link text
+        // Pattern explanation: [url text] --> text
+        content = content.replace(/\[http[s]?:\/\/[^\s]+ ([^\]]+)\]/g, '$1');
+    
+        // Remove Markdown link references, keeping only the link text
+        // Pattern explanation: ![link text](url) --> link text
+        content = content.replace(/\!\[([^\]]+)\]\([^\)]+\)/g, '$1');
+    
+        return content;
+    }
+
+    sanitizeBlocks(blocks: Array<any>) {
+        let sanitizedBlocks = <any>[] ;
+        blocks.forEach(block => {
+            let sanitizedBlock: any = {};
+            for (let key in block) {
+                let value = block[key];
+                if (typeof value === 'string') {
+                    sanitizedBlock[key] = this.sanitizeWikiContent(value);
+                } else if (Array.isArray(value)) {
+                    sanitizedBlock[key] = this.sanitizeBlocks(value);
+                } else {
+                    sanitizedBlock[key] = value;
+                }
+            }
+            sanitizedBlocks.push(sanitizedBlock);
+        });
+        return sanitizedBlocks;
+    }
+    
+    
+}

package/src/class/services/open_ai_service.ts

@@ -0,0 +1,18 @@
+import { openAIRequest } from "../../service/open_ai_request";
+
+
+export class OpenAiService{
+    public api_key: string;
+    public model: string;
+    
+
+    constructor(apiKey: string, model:string){
+        this.api_key = apiKey;
+        this.model =model
+    }
+
+    async sendRequest(prompt: string,content: string){
+        return await openAIRequest(content,prompt,this.api_key,this.model);
+    } 
+}
+

package/src/config.ts

@@ -0,0 +1,7 @@
+import "dotenv/config";
+
+const config = {
+    openAIKey  : process.env.OPEN_AI_KEY as string
+}
+
+export default config;