only_ever_generator 1.0.0 → 1.0.1

package/dist/bootstrap/app.js

@@ -1,16 +1,85 @@
 "use strict";
+// import { ParseSourceContent } from "../class/parse/parse_source_content";
 // import { OpenAiService } from "../class/services/open_ai_service";
 // export class OnlyEverGenerator{
 //     public api_key: string = '';
 //     public openAiService: OpenAiService | undefined;
-//     constructor(apiKey:string){
+//     parsedContent: String = '';
+//     constructor(apiKey:string, content: Array<any>){
 //         this.api_key = apiKey;
 //         this.openAiService = new OpenAiService(apiKey);
+//         this.parsedContent = new ParseSourceContent(content).parse();
 //     };
+//     typologyResponse = {};
+//     cardgenResponse = {};
+//     summarizeResponse = {};
+//     async generate( 
+//         args: GenerateArgs
+//     ): Promise<Array<any>> {
+//         const responseToReturn = [];
+//         const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
+//         for(let elem of whatNeedsToBeGenerated)
+//         if(elem == 'generate_tyopology'){
+//             this.typologyResponse = await this.generateTypology(args.prompts.typology_prompt ?? '', this.parsedContent);
+//             responseToReturn.push(this.typologyResponse);
+//         }else if(elem == 'generate_card'){
+//             this.cardgenResponse =  await this.generateCard(args.prompts.card_gen_prompt ?? '', this.parsedContent +  JSON.stringify(this.typologyResponse));
+//             responseToReturn.push(this.cardgenResponse);
+//         }else if(elem == 'generate_summary'){
+//             this.summarizeResponse = await this.generateSummary(args.prompts.summary_prompt ?? '', this.parsedContent);
+//             responseToReturn.push(this.summarizeResponse);
+//         }
+//         return responseToReturn;
+//     }
+//     returnParsedContent(){
+//         return this.parsedContent;
+//     }
 //     async generateCard(prompt: String, content: String){
-//         return await this.openAiService?.sendRequest(prompt,content);
+//         let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
+//         response['type'] = 'card_gen';
+//         return response;
 //     }
 //     async generateTypology(prompt:String,content:String){
-//         return await this.openAiService?.sendRequest(prompt,content);
+//         let response =  await this.openAiService?.sendRequest(prompt,this.parsedContent);
+//         response['type'] = 'typology';
+//         return response;
+//     }
+//     async generateSummary(prompt:String,content:String){
+//         let response = await this.openAiService?.sendRequest(prompt,this.parsedContent);
+//         response['type']= 'summary';
+//         return response;
+//     }
+// }
+// export class GenerateArgs{
+//     public generate_card : boolean = false;
+//     public generate_typology: boolean = false;
+//     public generate_summary: boolean = false;
+//     public prompts = {
+//         typology_prompt: '',
+//         card_gen_prompt: '',
+//         summary_prompt: ''
+//     }
+//     constructor(generate_card: boolean,generate_typology: boolean, generate_summary: boolean, prompts = {
+//         typology_prompt: '',
+//         card_gen_prompt: '',
+//         summary_prompt: ''
+//     }){
+//         this.generate_card = generate_card;
+//         this.generate_typology = generate_typology;
+//         this.generate_summary = generate_summary;
+//         this.prompts = prompts
+//     }
+//     getWhatNeedsToBeGenerated(){
+//         let returnData = [];
+//         if(this.generate_typology == true){
+//             returnData.push('generate_tyopology')
+//         }
+//         if(this.generate_summary == true){
+//             returnData.push('generate_summary')
+//         }
+//         if(this.generate_card == true){
+//             returnData.push('generate_card');
+//         }
+//         return returnData;
 //     }
 // }

package/dist/class/parse/parse_source_content.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParseSourceContent = void 0;
+class ParseSourceContent {
+    constructor(sourceContent) {
+        this.titles_to_remove = ['See also', 'References', 'Further reading', 'External links', 'Notes and references', 'Bibliography', 'Notes', 'Cited sources'];
+        this.content = sourceContent;
+    }
+    parse() {
+        let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(this.content);
+        let afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
+        return JSON.stringify(afterSanitized);
+    }
+    removeSectionsByTitle(data) {
+        let dataAfterRemoving = [];
+        for (let elem of data) {
+            if (elem.block_type == 'heading' && this.titles_to_remove.includes(elem.content)) {
+                continue;
+            }
+            if (elem.children) {
+                elem.children = this.removeSectionsByTitle(elem.children);
+            }
+            dataAfterRemoving.push(elem);
+        }
+        return dataAfterRemoving;
+    }
+    sanitizeWikiContent(content) {
+        // Remove newline characters
+        content = content.replace(/\\n/g, ' ');
+        // Remove internal link references, keeping only the link text
+        // Pattern explanation: [[link|text|index|wiki]] --> text
+        content = content.replace(/\[\[.*?\|(.*?)\|.*?\|wiki\]\]/g, '$1');
+        // Remove external links, keeping only the link text
+        // Pattern explanation: [url text] --> text
+        content = content.replace(/\[http[s]?:\/\/[^\s]+ ([^\]]+)\]/g, '$1');
+        // Remove Markdown link references, keeping only the link text
+        // Pattern explanation: ![link text](url) --> link text
+        content = content.replace(/\!\[([^\]]+)\]\([^\)]+\)/g, '$1');
+        return content;
+    }
+    sanitizeBlocks(blocks) {
+        let sanitizedBlocks = [];
+        blocks.forEach(block => {
+            let sanitizedBlock = {};
+            for (let key in block) {
+                let value = block[key];
+                if (typeof value === 'string') {
+                    sanitizedBlock[key] = this.sanitizeWikiContent(value);
+                }
+                else if (Array.isArray(value)) {
+                    sanitizedBlock[key] = this.sanitizeBlocks(value);
+                }
+                else {
+                    sanitizedBlock[key] = value;
+                }
+            }
+            sanitizedBlocks.push(sanitizedBlock);
+        });
+        return sanitizedBlocks;
+    }
+}
+exports.ParseSourceContent = ParseSourceContent;

package/dist/class/services/open_ai_service.js

@@ -12,12 +12,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAiService = void 0;
 const open_ai_request_1 = require("../../service/open_ai_request");
 class OpenAiService {
-    constructor(apiKey) {
+    constructor(apiKey, model) {
         this.api_key = apiKey;
+        this.model = model;
     }
     sendRequest(prompt, content) {
         return __awaiter(this, void 0, void 0, function* () {
-            return yield (0, open_ai_request_1.openAIRequest)(content, prompt, this.api_key);
+            return yield (0, open_ai_request_1.openAIRequest)(content, prompt, this.api_key, this.model);
         });
     }
 }

package/dist/constants/prompts/card_gen_prompt.js

@@ -2,74 +2,268 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.returnCardGenPrompt = void 0;
 const promptString = `
-Task Objective:
-Please generate an array of different types of question-and-answer pairs based on the provided content schema. 
-The types include flash cards, multiple choice questions (MCQs), and cloze cards. Each type has specific formatting requirements
-as detailed below. Each question should come from a different H1 heading, ensuring diversity in the topics covered. 
-Format the response as JSON, adhering to the following example structures for each type of card:
+As a dedicated assistant at a learning company, your role is to analyze educational content and create test cards that help learners understand and remember key concepts and facts. You will be provided with:
 
-Flash Cards
-Each flash card should relate to content under H1 headings or nested within their sections. The answers should be concise, limited to 
-a single line, and presented in plain text. The "heading" in the response should reference the title of the parent H1 heading.
-JSON Structure:
+1. Title of the source
+2. Main headings
+3. The content
+4. The field of knowledge it belongs
+5. Key concepts in the source
+6. Important facts in the source.
+7. Summary of the content using cards
 
+Follow these steps:
+
+1. Analyze the content to identify any key concepts missing from the provided list.
+2. Analyze the content to identify any important facts missing from the provided list.
+3. Generate test cards for each concept and fact, tethered to either the entire source or specific headings.
+4. Ensure all concepts and facts have at least one test card.
+
+Please format your response in the following format.
+json
+    "missing_concepts": ["concept1", "concept2", "concept3", "..."],
+    "missing_facts": ["fact1", "fact2", "fact3", "..."],
+    "test_cards": [
+        {
+            "type": "flash" | "mcq" | "cloze" | "match",
+            "card_content": { "front": "...", "back": "..." | "prompt": "...", "choices": [ ... ] | "prompt": "...", "options": [ ... ] | "right_choice 1": "...", "left_choice 1": "..." },
+            "card_reference": "source_title#heading",
+            "concepts": ["Concept1", "Concept2", "..."],
+            "facts": ["Fact1", "Fact2", "..."]
+        }
+    ]
+
+**Criteria**
+•	Each test card must include at least one concept or fact.
+•	Flashcards must not exceed 15% of the total number of cards.
+•	Each concept and fact must have at least one test card.
+
+Further instructions are provided below.
+Concepts are fundamental ideas that form the basis of knowledge in any discipline. They help organize and explain information, making it accessible and relatable.
+
+You are provided with a list of identified concepts. Review this list and the content to determine if any concepts are missing.
+
+1. **Definition of a Concept**: Concepts should be significant ideas within the content that are essential for understanding the main themes.
+2. **Inclusion Criteria**: Include a concept only if it has not been previously included in the list provided to you.
+
+List the concepts in the following JSON format:
+
+json
+{
+    "missing_concepts": ["concept1", "concept2", "concept3", "..."]
+}
+
+Facts are objective statements supported by empirical evidence or observation, such as data on events, people, numbers, dates, or well-established ideas.
+
+You are provided with a list of identified facts. Review this list and the content to determine if any facts are missing.
+
+1. **Definition of a Fact**: Standalone information that is concrete and independently verifiable.
+2. **Selection Criteria**: Choose facts based on their significance to the content's main themes or concepts, their educational value, or their foundational role in the subject. Only inlcude those that have not present in the list provided to you.
+
+Record the facts in the following JSON format:
+
+json
 {
-  "flash_cards": [
-    {
-      "question": "What is the primary function of the 'XYZ' system introduced in the first section?",
-      "answer": "The 'XYZ' system primarily facilitates automated data analysis.",
-      "heading": "XYZ System Overview"  // note: this can also be empty
+    "missing_facts": ["fact1", "fact2", "fact3", "..."]
+}
+
+After you have the complete list of concepts and facts, including any missing ones you identified, proceed to generate test cards for each.
+
+	1.	Clarity: Ensure the test content is clear and unambiguous.
+	2.	Specificity: Be specific about what you are asking. Avoid vague or overly broad questions or prompts.
+	3.	Simplicity: Use simple and direct language. Avoid complex sentences or jargon unless testing understanding of that jargon.
+	4.	Relevance: Ensure the test content is directly related to the key concepts or facts you want to test.
+
+Test cards must be one of the following types:
+
+1.	Flashcards: Have a front and back. 
+
+json
+{
+    "type": "flash",
+    "card_content": {
+        "front": "<content for the front>",
+        "back": "<content for the back>"
     },
-    // Add more flash card question-and-answer pairs here, each from a different H1 heading
-  ]
+    "card_reference": "source_title#heading",
+    "concepts": ["Concept1", "Concept2", "..."],
+    "facts": ["Fact1", "Fact2", "..."]
 }
 
-Multiple Choice Questions (MCQs)
-Each MCQ should have up to six options, with at least one correct answer and at most six correct answers. 
-Ensure that each choice or answer does not exceed 24 characters. The answers list should be a map in JSON format but represented as a string.
-The "heading" in the response should reference the title of the parent H1 heading.
-JSON Structure:
+- Each side must not exceed 300 characters.
+2. Multiple Choice Questions (MCQ): Provide multiple choices to pick from. One or more should be correct.
+
+json
+{
+    "type": "mcq",
+    "card_content": {
+        "prompt": "<question text>",
+        "choices": [
+            {"choice": "choice 1", "is_correct": true or false},
+            {"choice": "choice 2", "is_correct": true or false},
+            "... up to 8 choices"
+        ]
+    },
+    "card_reference": "source_title#heading",
+    "concepts": ["Concept1", "Concept2", "..."],
+    "facts": ["Fact1", "Fact2", "..."]
+}
+
+•	Minimum choices required: 2
+•	Maximum choices allowed: 8
+•	Minimum correct choices required: 1
+•	Maximum character length for the prompt: 320
+•	Maximum character length for each choice: 42
+
+3.	Cloze: Fill-in-the-blank style test card. Use double curly braces {{}} to indicate a cloze.
+
+json
 {
-  "mcqs": [
-    {
-      "question": "What is the primary function of the 'XYZ' system introduced in the first section?",
-      "answers": [
-        {"answer": "Data analysis", "is_correct": true},
-        {answer: "User auth", "is_correct": false}",
-        {"answer": "Data security", "is_correct": false},
-        {"answer": "Real-time viz", "is_correct": false}
-      ],
-      "heading": "XYZ System Overview"
+    "type": "cloze",
+    "card_content": {
+        "prompt": "Accidentals in music denote {{c0:notes}} that do not belong to the {{c1:scale}} or {{c2:mode}} indicated by the key signature.",
+        "options": [
+            {"option": "notes", "cloze": "c0"},
+            {"option": "scale", "cloze": "c1"},
+            {"option": "mode", "cloze": "c2"},
+            {"option": "chords", "cloze": "null"},
+            "... up to 8 choices"
+        ]
     },
-    // Add more multiple choice question-and-answer pairs here, each from a different H1 heading
-  ]
+    "card_reference": "source_title#heading",
+    "concepts": ["Concept1", "Concept2", "..."],
+    "facts": ["Fact1", "Fact2", "..."]
 }
 
-Cloze Cards
-Each cloze question should have up to six options to fill in the statement, with at least one correct answer and at most six correct answers.
-The options field should be a list of choices in the form of a map, where the option key holds the actual choice and the cloze key holds the 
-position of that option. If the cloze key is null, the option is an incorrect option. The "heading" in the response should reference the title of the parent H1 heading.
-On the question the clozes should be in format {{c0:shreya}}.
-JSON Structure:
+•	Minimum choices required: 2
+•	Maximum choices allowed: 8
+•	Minimum correct choices required: 1
+•	Maximum character length for the prompt: 320
+•	Maximum character length for an individual cloze: 90
 
+4.	Match: Pairing items.
+
+json
 {
-  "cloze_cards": [
-    {
-      "question": "My name is {{c0:shreya}} my last name is {{c1:baidya}}",
-      "options": [
-        {"option":"shreya","cloze":"c0"},
-        {"option":"baidya","cloze":"c1"},
-        {"option":"Shreedhar","cloze":null},
-        {"option":"Pandey","cloze":null},
-        {"option":"Tuppi","cloze":null},
-        {"option":"Bajracharya","cloze":null}
-      ]
+    "type": "match",
+    "card_content": {
+        "right_choice 1": "left_choice 1",
+        "right_choice 2": "left_choice 2",
+        "... up to 8 total pairs"
     },
-    // Add more cloze question-and-answer pairs here, each from a different H1 heading
-  ]
+    "card_reference": "source_title#heading",
+    "concepts": ["Concept1", "Concept2", "..."],
+    "facts": ["Fact1", "Fact2", "..."]
+}
+
+•	Maximum character length for each item in a pair: 42
+
+
+The schema for each card type is below.
+
+A flashcard consists of two sides: a front and a back. Both the front and back text content must not exceed more than 300 characters in length.
+
+The content schema should be represented in this manner:
+json
+{
+    "front": "<content for the front of the flashcard>",
+    "back": "<content for the back of the flashcard>"
 }
 
-Try to generate as much as possible.
+
+The schema for an mcq card is shown below.
+
+json
+{
+    "prompt": "<question text>",
+    "choices": 
+    [
+        {
+            choice: "choice 1",
+            is_correct: true or false
+        },
+        {
+            choice: "choice 2",
+            is_correct: true or false
+        }
+        "... up to 8 choices"
+    ]
+}
+
+Minimum choices required: 2
+Maximum choices allowed: 8
+Minimum correct choices required: 1
+Maximum character length for the prompt: 320
+Maximum character length for each choice: 42
+
+
+For questions of the "type": "cloze", which refers to a fill-in-the-blank style question, the format is outlined below. I have used the sample text here because it is easier to illustrate the schema with an example:
+
+json
+{
+    "prompt": "Accidentals in music denote {{c0:notes}} that do not belong to the {{c1:scale}} or {{c2:mode}} indicated by the key signature."
+    "options": 
+    [
+        {
+            "option: "notes",
+            "cloze": "c0"
+        },
+        {
+            "option: "scale",
+            "cloze": "c1"
+        },
+        {
+            "option: "mode",
+            "cloze": "c2"
+        },
+        {
+            "option: "chords",
+            "cloze": "null"
+        },
+        {
+            "option: "tilda",
+            "cloze": "null"
+        },
+        {
+            "option: "score",
+            "cloze": "null"
+        },
+        "... up to a maximum of 8 choices total"
+    ]
+}
+
+Minimum choices required: 2
+Maximum choices allowed: 8
+Minimum correct choices required: 1
+Maximum character length for the prompt: 320
+Maximum character length for an individual cloze: 90
+
+The schema for match type cards are as follows.
+json
+{
+    "right_choice 1": "left_choice 1",
+    "right_choice 2": "left_choice 2"
+    "right_choice 3": "left_choice 3"
+    "... up to 8 total pairs"
+}
+
+Maximum character length for each item in a pair: 42
+Each test card needs a reference. A reference can either be the entire source or a specific heading in the source. Whenever possible, pick a main heading to direct the user to the most relevant part of the source material. The reference schema is as follows: source_title#main_heading, where #main_heading is optional.
+PART 6
+
+Once you are done generating the test cards. Go back and evaulate the full list of concepts and fact that include any of the missing concepts or facts along with the list that was provided as the input. 
+
+Are there any concept or fact that don't have a test card yet? If yes, go back and create one.
+
+Once you are done creating come back to this step again to check if you have full coverage of all the concepts and facts in the source. You can stop generating test questions once you achieve full coverage.
+
+Once you are done generating the test cards, review the full list of concepts and facts, including any missing ones you identified.
+
+	1.	Ensure every concept and fact has at least one test card (if not more).
+	2.	If any concept or fact is missing a test card, create one for it.
+	3.	Repeat this step until all concepts and facts are covered.
+
+Only stop generating test questions once you believe there is sufficient testing material for learners to fully understand the concepts and remember the facts. The same concept or fact can have multiple test cards, so continue creating test cards until you are confident that there are enough for learners to fully grasp the source material.
 `;
 function returnCardGenPrompt() {
     return promptString;

package/dist/constants/prompts/typology_prompt.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.returnTypologyPrompt = void 0;
 const typologyPromptString = `
-As a dedicated assistant at a learning company, your role involves analyzing educational content to enhance learning efficiency. You will process content (in JSON format) that represents text and images from diverse sources such as PDFs, book chapters, videos, and websites. Follow these steps:
+As a dedicated assistant at a learning company, your role involves analyzing educational content to categorize and summarize it. You will process content (in JSON format) that represents text and images from diverse sources such as PDFs, book chapters, videos, and websites. Follow these steps:
 
 1. Classify the content into one to three predefined fields of knowledge.
 2. Identify key concepts within the content.
@@ -12,16 +12,15 @@ As a dedicated assistant at a learning company, your role involves analyzing edu
 
 Please format your findings in this JSON schema:
 json
-    {
-        "field": ["primary_field", "secondary_field", "tertiary_field"],
-        "concepts": ["concept1", "concept2", "concept3", "..."],
-        "facts": ["fact1", "fact2", "fact3", "..."],
-        "generate_cards": true or false
-        "summary_cards": ["summary_card1", "summary_card2", "summary_card3", "..."]
-    }
+{
+    "field": ["primary_field", "secondary_field", "tertiary_field"],
+    "concepts": ["concept1", "concept2", "concept3", "..."],
+    "facts": ["fact1", "fact2", "fact3", "..."],
+    "generate_cards": true or false
+    "summary_cards": ["summary_card1", "summary_card2", "summary_card3", "..."]
+}
 
 Further instruction on how to perform these tasks are below.
-
 Every source must be placed under a field. This is the broadest category of knowledge. A source should belong to at least one and at most 3 fields. Only include fields that a source is strongly associated with. The field names in your response must exactly match the names of 18 fields listed below.
 
 1.	Sciences: Focus on Biology, Chemistry, Physics, Astronomy, Mathematics, and Computer Science.
@@ -64,7 +63,8 @@ json
 {
     "facts": ["fact1", "fact2", "fact3", "..."]
 }
-    After analyzing the content, classifying its field, and identifying key concepts, and facts, assess whether the discovered elements warrant the creation of testing materials. Consider if these elements provide significant educational value to an average learner by enhancing understanding, offering practical applications, or supporting crucial educational goals.
+
+After analyzing the content, classifying its field, and identifying key concepts, and facts, assess whether the discovered elements warrant the creation of testing materials. Consider if these elements provide significant educational value to an average learner by enhancing understanding, offering practical applications, or supporting crucial educational goals.
 
 1. **Value Assessment**: Determine if the concepts and facts are essential for understanding the broader topic, are likely to be used in practical scenarios, or help in achieving educational benchmarks.
 2. **Criteria for Material Generation**: Generate testing materials if the concepts and facts are central to the content, have broad applicability, and are likely to reinforce or expand the learner’s knowledge significantly.
@@ -74,7 +74,8 @@ json
 {
     "generate_cards": true or false
 }
-    After analyzing the content, identifying key concepts, and facts, summarize the material using a series of engaging and informative cards. These cards should capture the essence of the content while highlighting the critical concepts and facts that you previously identified.
+
+After analyzing the content, identifying key concepts, and facts, summarize the material using a series of engaging and informative cards. These cards should capture the essence of the content while highlighting the critical concepts and facts that you previously identified.
 
 
 1. **Inclusion Criteria**: The generate_cards should be true. Return an empty array if the generate_cards is false.
@@ -87,8 +88,6 @@ json
 {
     "summary_cards": ["summary_card1", "summary_card2", "summary_card3", "..."]
 }
-
-Calculate the tokens that will be used, and try to generate completions with atleast more than 4k tokens
 `;
 function returnTypologyPrompt() {
     return typologyPromptString;