only_ever_generator 1.0.3 → 1.0.5

package/src/parse/parse_card_response.ts

@@ -0,0 +1,117 @@
+import { ErrorLogger } from "../logger";
+import { ParseClozeCard } from "./parse_card/parse_cloze_card";
+import { ParseFlashCard } from "./parse_card/parse_flash_cards";
+import { ParseMatchCard } from "./parse_card/parse_match_card";
+import { ParseMcqCard } from "./parse_card/parse_mcq_card";
+
+export class ParseCardResponse {
+  parse(generatedData: any, isGapFill: boolean, sourceTaxonomy: any) {
+    let usage_data = generatedData.metadata;
+    try {
+      const cardData = [];
+      const unparsedTestCards = generatedData.generated_content.test_cards;
+      const type = generatedData.type;
+
+      if (unparsedTestCards !== undefined && unparsedTestCards.length != 0) {
+        for (let elem of unparsedTestCards) {
+          const managedCardConcepts = this._mapIdToConcepts(
+            elem.concepts_facts,
+            sourceTaxonomy.concepts_facts
+          );
+          elem.concepts = managedCardConcepts;
+          if (elem.type == "flash") {
+            const flashCard = new ParseFlashCard().parse(elem);
+            if (flashCard != null && flashCard) {
+              flashCard.heading = this._getCardReference(
+                flashCard,
+                sourceTaxonomy
+              );
+
+              cardData.push(flashCard);
+            }
+          } else if (elem.type == "mcq") {
+            const mcqCard = new ParseMcqCard().parse(elem);
+            if (mcqCard != null && mcqCard) {
+              mcqCard.heading = this._getCardReference(mcqCard, sourceTaxonomy);
+              cardData.push(mcqCard);
+            }
+          } else if (elem.type == "cloze") {
+            const clozeCard = new ParseClozeCard().parse(elem);
+            if (clozeCard && clozeCard != null) {
+              clozeCard.heading = this._getCardReference(
+                clozeCard,
+                sourceTaxonomy
+              );
+              cardData.push(clozeCard);
+            }
+          } else if (elem.type == "match") {
+            const matchCard = new ParseMatchCard().parse(elem);
+            if (matchCard && matchCard != null) {
+              matchCard.heading = this._getCardReference(
+                matchCard,
+                sourceTaxonomy
+              );
+              cardData.push(matchCard);
+            }
+          }
+        }
+      } else {
+        if (!isGapFill) {
+          usage_data.status = "failed";
+        }
+      }
+      if (cardData.length == 0) {
+        usage_data.status = "failed";
+      }
+
+      return {
+        status_code: cardData.length == 0 ? 400 : 200,
+        metadata: usage_data,
+        type: type,
+
+        cards_data: cardData,
+      };
+    } catch (e: any) {
+      return {
+        status_code: 500,
+        metadata: usage_data,
+        type: generatedData.type,
+      };
+    }
+  }
+
+  _getCardReference(generatedCardData: any, sourceTaxonomy: any) {
+    const cardConcepts = (generatedCardData.concepts ?? []).map(
+      (e: any) => e.text
+    );
+
+    const sourceConceptsFacts = sourceTaxonomy.concepts_facts ?? [];
+    const firstMatchedConcept = sourceConceptsFacts.find((elem: any) =>
+      cardConcepts.includes(elem.text)
+    );
+
+    if (firstMatchedConcept) {
+      return firstMatchedConcept.reference;
+    } else {
+      return "";
+    }
+  }
+
+  _mapIdToConcepts(cardConcepts: any[], sourceConceptsFacts: any[]) {
+    // const cardData = cardC
+    const managedCardConcepts = [];
+    for (const cardConcept of cardConcepts) {
+      const matchedConcept = sourceConceptsFacts.find(
+        (e: any) =>
+          e.text.toLowerCase().trim() == cardConcept.text.toLowerCase().trim()
+      );
+      if (matchedConcept) {
+        managedCardConcepts.push({
+          id: matchedConcept.id,
+          text: matchedConcept.text,
+        });
+      }
+    }
+    return managedCardConcepts;
+  }
+}

package/src/parse/parse_source_content.ts

@@ -0,0 +1,202 @@
+export class ParseSourceContent {
+  public content: any;
+  /// Format of Content
+  // content: {
+  //     title: source.title,
+  //     headings: source.headings,
+  //     content: source.content,
+  //     fields: fields,
+  //     taxonomy: source.source_taxonomy,
+  //     type: source.source_type
+  //   },
+
+  titles_to_remove = [
+    "See also",
+    "References",
+    "Further reading",
+    "External links",
+    "Notes and references",
+    "Bibliography",
+    "Notes",
+    "Cited sources",
+  ];
+  block_types_toremove = ["table", "empty_line"];
+  constructor(sourceContent: any) {
+    this.content = sourceContent;
+  }
+
+  parseData() {
+    let sourceType = this.content.type;
+    let afterSanitized;
+    if (sourceType == "video") {
+      afterSanitized = this.parseVideoContent(this.content.content);
+    } else {
+      let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(
+        this.content.content
+      );
+      afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
+    }
+    return {
+      type: this.content.type,
+      title: this.content.title,
+      content: afterSanitized,
+      headings: this.content.headings,
+      taxonomy: this.content.taxonomy,
+    };
+  }
+
+  removeSectionsByTitle(data: Array<any>) {
+    let dataAfterRemoving = [];
+    for (let elem of data) {
+      if (
+        elem.block_type == "heading" &&
+        this.titles_to_remove.includes(elem.content)
+      ) {
+        continue;
+      }
+      /// remove unwanted blcok types , for now `table` and `empty_line`
+      if (this.block_types_toremove.includes(elem.block_type)) {
+        continue;
+      }
+      if (elem.children) {
+        elem.children = this.removeSectionsByTitle(elem.children);
+      }
+      dataAfterRemoving.push(elem);
+    }
+    return dataAfterRemoving;
+  }
+
+  sanitizeTextContent(content: String) {
+    // Remove newline characters
+    content = content.replace(/\\n/g, " ");
+
+    // Remove internal link references, keeping only the link text
+    // Pattern explanation: [[link|text|index|wiki]] --> text
+    content = content.replace(/\[\[.*?\|(.*?)\|.*?\|wiki\]\]/g, "$1");
+
+    // Remove external links, keeping only the link text
+    // Pattern explanation: [url text] --> text
+    content = content.replace(/\[http[s]?:\/\/[^\s]+ ([^\]]+)\]/g, "$1");
+
+    // Remove Markdown link references, keeping only the link text
+    // Pattern explanation: ![link text](url) --> link text
+    content = content.replace(/\!\[([^\]]+)\]\([^\)]+\)/g, "$1");
+
+    return content;
+  }
+
+  sanitizeBlocks(blocks: Array<any>) {
+    let sanitizedBlocks = <any>[];
+    blocks = blocks.filter((item) => item.block_type != "table");
+    blocks.forEach((block) => {
+      let sanitizedBlock: any = {};
+      for (let key in block) {
+        let value = block[key];
+        if (typeof value === "string") {
+          sanitizedBlock[key] = this.sanitizeTextContent(value);
+        } else if (Array.isArray(value)) {
+          sanitizedBlock[key] = this.sanitizeBlocks(value);
+        } else {
+          sanitizedBlock[key] = value;
+        }
+      }
+      sanitizedBlocks.push(sanitizedBlock);
+    });
+    return sanitizedBlocks;
+  }
+
+  parseVideoContent(data: Array<any>) {
+    let finalChapters: Array<any> = [];
+    // let cleanedData = this.cleanTranscript(timeCodes);
+    data.forEach((e) => {
+      let combinedContent = this.cleanTranscript(e);
+      finalChapters.push({
+        startTime: e.startTime,
+        endTime: e.endTime,
+        content: combinedContent,
+        title: e.content,
+      });
+    });
+
+    return finalChapters;
+  }
+
+  // remove content inside [] which denotes non-speech sounds
+  isNonSpeech(content: string) {
+    // Check if the content is non-speech (enclosed in square brackets).
+    return /^\[.*\]$/.test(content.trim());
+  }
+
+  // remove non-essential content
+  cleanTranscript(data: any) {
+    let finalContent = "";
+    let children = data.children ?? [];
+
+    children.forEach((e: any) => {
+      let content = (e.content || "").trim();
+
+      if (this.isNonSpeech(content)) return;
+
+      content = content.replace(/\s+/g, " ");
+      finalContent += content;
+    });
+
+    return finalContent;
+  }
+
+  // collapse the timecode to 30 seconds
+  collapseTimeCodes(data: Array<any>, maxDuration = 30.0) {
+    // Collapse time codes into buckets of approximately maxDuration seconds.
+    const collapsedData = [];
+    let bucketStartTime: number | null = null;
+    let bucketEndTime: number | null = null;
+    let bucketContent: Array<any> = [];
+    let bucketDuration = 0.0;
+
+    data.forEach((entry) => {
+      const startTime = entry.start_time;
+      const endTime = entry.end_time;
+      const content = entry.content;
+      const entryDuration = endTime - startTime;
+
+      if (bucketStartTime === null) {
+        // Start a new bucket
+        bucketStartTime = startTime;
+        bucketEndTime = endTime;
+        bucketContent.push(content);
+        bucketDuration = entryDuration;
+      } else if (bucketDuration + entryDuration <= maxDuration) {
+        // Add to current bucket
+        bucketEndTime = endTime;
+        bucketContent.push(content);
+        bucketDuration += entryDuration;
+      } else {
+        // Close current bucket and start a new one
+        const collapsedEntry = {
+          start_time: bucketStartTime,
+          end_time: bucketEndTime,
+          content: bucketContent.join(" "),
+        };
+        collapsedData.push(collapsedEntry);
+
+        // Start new bucket with current entry
+        bucketStartTime = startTime;
+        bucketEndTime = endTime;
+        bucketContent = [content];
+        bucketDuration = entryDuration;
+      }
+    });
+
+    // Add the last bucket if it exists
+    if (bucketContent.length > 0) {
+      const collapsedEntry = {
+        start_time: bucketStartTime,
+        end_time: bucketEndTime,
+        content: bucketContent.join(" "),
+      };
+      collapsedData.push(collapsedEntry);
+    }
+
+    return collapsedData;
+  }
+}

package/src/parse/response_format_card.ts

@@ -0,0 +1,386 @@
+const responseData = {
+  flash_cards: [
+    {
+      question: "What is the primary function of the 'Electrolysis'?",
+      answer:
+        "Electrolysis is the passing of a direct electric current through an electrolyte producing chemical reactions at the electrodes and decomposition of the materials.",
+      heading: "Overview",
+    },
+    {
+      question:
+        "What did Michael Faraday discover while studying the process of electrolysis under Humphry Davy?",
+      answer: "Michael Faraday discovered two laws of electrolysis.",
+      heading: "History",
+    },
+    {
+      question:
+        "What can electrolysis be used for in the manufacturing sector?",
+      answer:
+        "Electrolysis can be used for electroplating and electrochemical machining.",
+      heading: "Manufacturing processes",
+    },
+    {
+      question:
+        "In the context of electrolysis, what does the term 'Decomposition potential' refer to?",
+      answer:
+        "Decomposition potential or decomposition voltage refers to the minimum voltage between anode and cathode of an electrolytic cell needed for electrolysis to occur.",
+      heading: "Decomposition potential",
+    },
+  ],
+  mcqs: [
+    {
+      question: "What is the primary purpose of 'Electrometallurgy'?",
+      answers: [
+        {
+          answer: "Production of aluminium",
+          is_correct: true,
+        },
+        {
+          answer: "Chlorine production",
+          is_correct: false,
+        },
+        {
+          answer: "Purifying copper",
+          is_correct: false,
+        },
+        {
+          answer: "Rust removal",
+          is_correct: false,
+        },
+        {
+          answer: "Hydrogen production",
+          is_correct: false,
+        },
+      ],
+      heading: "Industrial uses",
+    },
+    {
+      question:
+        "What can electrolysis be used for in the context of batteries?",
+      answers: [
+        {
+          answer: "Spontaneous redox reactions",
+          is_correct: true,
+        },
+        {
+          answer: "Energy-releasing reactions",
+          is_correct: false,
+        },
+        {
+          answer: "Disinfectant production",
+          is_correct: false,
+        },
+        {
+          answer: "Fuel production",
+          is_correct: false,
+        },
+        {
+          answer: "Gas diffusion in reactors",
+          is_correct: false,
+        },
+      ],
+      heading: "Related processes",
+    },
+  ],
+  cloze_cards: [
+    {
+      question:
+        "Electrolysis is the passing of a {{c0:direct electric current}} through an {{c1:electrolyte}} producing {{c2:chemical reactions}} at the {{c3:electrodes}} and {{c4:decomposition}} of the materials.",
+      options: [
+        {
+          option: "direct electric current",
+          cloze: "c0",
+        },
+        {
+          option: "electrolyte",
+          cloze: "c1",
+        },
+        {
+          option: "chemical reactions",
+          cloze: "c2",
+        },
+        {
+          option: "electrodes",
+          cloze: "c3",
+        },
+        {
+          option: "decomposition",
+          cloze: "c4",
+        },
+        {
+          option: "metallic objects",
+          cloze: null,
+        },
+      ],
+    },
+    {
+      question:
+        "In electrolysis, the decomposition potential or decomposition voltage refers to the minimum voltage between {{c0:anode}} and {{c1:cathode}} of an electrolytic cell needed for electrolysis to occur.",
+      options: [
+        {
+          option: "anode",
+          cloze: "c0",
+        },
+        {
+          option: "cathode",
+          cloze: "c1",
+        },
+        {
+          option: "electrolyte",
+          cloze: null,
+        },
+        {
+          option: "chemical reactions",
+          cloze: null,
+        },
+        {
+          option: "oxygen",
+          cloze: null,
+        },
+        {
+          option: "hydrogen",
+          cloze: null,
+        },
+      ],
+    },
+  ],
+};
+
+const sourceResp = {
+  status_code: 200,
+  usage_data: {
+    prompt_tokens: 9781,
+    completion_tokens: 1915,
+    total_tokens: 11696,
+  },
+  generated_content: {
+    missing_concepts: [
+      "Faraday's laws of electrolysis",
+      "Humphry Davy",
+      "Electrolytic cell",
+      "Decomposition potential",
+      "Oxidation",
+      "Reduction",
+      "Electrolysis of seawater",
+      "Electrometallurgy",
+      "Electroplating",
+      "Electrochemical machining",
+      "Electrochemistry",
+      "Electrocatalysis",
+      "Electrorefining",
+      "Electrolysis of carbon dioxide",
+      "Energy changes during electrolysis",
+      "Electrocrystallization",
+      "Electrolysis of Iron Ore",
+    ],
+    missing_facts: [
+      "Michael Faraday introduced the term 'electrolysis' in 1834",
+      "The first demonstration of key electrolysis concepts was by William Nicholson and Anthony Carlisle in the early nineteenth century",
+      "Humphry Davy discovered several alkali and alkaline earth metals by electrolysis",
+      "Electrolysis played a key role in isolating and identifying new elements like lithium, chlorine, and fluorine",
+      "The Hall–Héroult process led to a significant drop in the price of aluminum",
+      "Hydrogen and oxygen are produced in a 2:1 ratio by the electrolysis of water",
+      "The electrolysis of seawater can result in the production of alkali hydroxides",
+      "Electrometallurgy is used in the production of various metals",
+      "Electroplating involves the deposition of a thin metal film onto a substrate material",
+      "Electrochemical machining is used for deburring or etching metal surfaces",
+      "Electrocatalysis involves the acceleration of electrochemical reactions",
+      "Electrorefining is used to obtain pure metals from impure ones",
+      "The electrolysis of carbon dioxide can produce methane, ethylene, or ethanol",
+      "Energy changes during electrolysis involve the addition of electrical energy, equal to the change in Gibbs free energy plus system losses",
+      "Electrocrystallization is a method for obtaining conductive crystals using electrolysis",
+      "Electrolysis of Iron Ore can eventually result in the reduction of emissions from steel production",
+    ],
+    test_cards: [
+      {
+        type: "flash",
+        card_content: {
+          front: "Who introduced the term 'electrolysis' and in which year?",
+          back: "Michael Faraday introduced the term 'electrolysis' in 1834",
+        },
+        card_reference: "Electrolysis#Etymology",
+        concepts: [],
+        facts: ["Michael Faraday introduced the term 'electrolysis' in 1834"],
+      },
+      {
+        type: "flash",
+        card_content: {
+          front: "What was the significance of the Hall–Héroult process?",
+          back: "The Hall–Héroult process led to a significant drop in the price of aluminum",
+        },
+        card_reference: "Electrolysis#Industrial uses",
+        concepts: ["Hall–Héroult process"],
+        facts: [
+          "The Hall–Héroult process led to a significant drop in the price of aluminum",
+        ],
+      },
+      {
+        type: "mcq",
+        card_content: {
+          prompt: "What does the electrolysis of water produce?",
+          choices: [
+            {
+              choice: "Oxygen and carbon dioxide",
+              is_correct: false,
+            },
+            {
+              choice: "Hydrogen and oxygen",
+              is_correct: true,
+            },
+            {
+              choice: "Nitrogen and helium",
+              is_correct: false,
+            },
+          ],
+        },
+        card_reference: "Electrolysis#Process of electrolysis",
+        concepts: ["Electrometallurgy"],
+        facts: [
+          "Hydrogen and oxygen are produced in a 2:1 ratio by the electrolysis of water",
+        ],
+      },
+      {
+        type: "cloze",
+        card_content: {
+          prompt:
+            "Electrolysis is the passing of a direct electric current through an {{c0:electrolyte}} producing chemical reactions at the electrodes and decomposition of the materials.",
+          options: [
+            {
+              option: "electrolyte",
+              cloze: "c0",
+            },
+            {
+              option: "anode",
+              cloze: "null",
+            },
+            {
+              option: "cathode",
+              cloze: "null",
+            },
+          ],
+        },
+        card_reference: "Electrolysis#Overview",
+        concepts: [],
+        facts: [],
+      },
+      {
+        type: "match",
+        card_content: {
+          "right_choice 1": "Electrometallurgy",
+          "right_choice 2": "Electroplating",
+          "right_choice 3": "Electrochemical machining",
+          "right_choice 4": "Electrochemistry",
+          "right_choice 5": "Electrocatalysis",
+          "right_choice 6": "Electrorefining",
+          "right_choice 7": "Electrolysis of carbon dioxide",
+          "right_choice 8": "Energy changes during electrolysis",
+          Electrometallurgy:
+            "The process of producing metals using electricity",
+          Electroplating:
+            "Deposition of thin metal film onto a substrate material",
+          "Electrochemical machining": "Deburring or etching metal surfaces",
+          Electrochemistry:
+            "Study of the interchange of chemical and electrical energy",
+          Electrocatalysis: "Acceleration of electrochemical reactions",
+          Electrorefining: "Obtaining pure metals from impure ones",
+        },
+        card_reference: "Electrolysis#Industrial uses",
+        concepts: [
+          "Electrometallurgy",
+          "Electroplating",
+          "Electrochemical machining",
+          "Electrochemistry",
+          "Electrocatalysis",
+          "Electrorefining",
+          "Electrolysis of carbon dioxide",
+          "Energy changes during electrolysis",
+        ],
+        facts: [
+          "Electrometallurgy is used in the production of various metals",
+          "Electroplating involves the deposition of a thin metal film onto a substrate material",
+          "Electrochemical machining is used for deburring or etching metal surfaces",
+          "Electrochemistry is the study of the interchange of chemical and electrical energy",
+          "Electrocatalysis involves the acceleration of electrochemical reactions",
+          "Electrorefining is used to obtain pure metals from impure ones",
+          "The electrolysis of carbon dioxide can produce methane, ethylene, or ethanol",
+          "Energy changes during electrolysis involve the addition of electrical energy, equal to the change in Gibbs free energy plus system losses",
+        ],
+      },
+      {
+        type: "mcq",
+        card_content: {
+          prompt: "What is a key application of electrolysis?",
+          choices: [
+            {
+              choice: "Production of metal from ore",
+              is_correct: false,
+            },
+            {
+              choice: "Pulsing current results",
+              is_correct: false,
+            },
+            {
+              choice: "Generating electrical potential",
+              is_correct: false,
+            },
+            {
+              choice: "Production of chlorine and sodium hydroxide",
+              is_correct: true,
+            },
+          ],
+        },
+        card_reference: "Electrolysis#Industrial uses",
+        concepts: ["Electrochemistry", "Electrocatalysis", "Electrorefining"],
+        facts: [
+          "Production of chlorine and sodium hydroxide, called the Chloralkali process",
+        ],
+      },
+      {
+        type: "cloze",
+        card_content: {
+          prompt:
+            "Oxidation of ions or neutral molecules occurs at the {{c0:anode}}. Reduction of ions or neutral molecules occurs at the {{c1:cathode}}.",
+          options: [
+            {
+              option: "anode",
+              cloze: "c0",
+            },
+            {
+              option: "cathode",
+              cloze: "c1",
+            },
+            {
+              option: "electrolyte",
+              cloze: "null",
+            },
+          ],
+        },
+        card_reference:
+          "Electrolysis#Oxidation and reduction at the electrodes",
+        concepts: ["Oxidation", "Reduction"],
+        facts: [],
+      },
+      {
+        type: "match",
+        card_content: {
+          "right_choice 1": "Electrolysis",
+          "right_choice 2": "Electrocrystallization",
+          "right_choice 3": "Electrolysis of Iron Ore",
+          "right_choice 4": "Electrolysis of seawater",
+          "right_choice 5": "Electrometallurgy",
+        },
+        card_reference: "Electrolysis#Research trends",
+        concepts: [
+          "Electrolysis",
+          "Electrocrystallization",
+          "Electrolysis of Iron Ore",
+          "Electrolysis of seawater",
+          "Electrometallurgy",
+        ],
+        facts: [],
+      },
+    ],
+  },
+  generated_at: 1718625081,
+  type: "card_gen",
+};