only_ever_generator 0.5.4 → 0.5.6

package/src/parse/parse_source_content.ts

@@ -1,197 +1,202 @@
-export class ParseSourceContent{
-    public content: any;
-    /// Format of Content
-    // content: {
-    //     title: source.title,
-    //     headings: source.headings,
-    //     content: source.content,
-    //     fields: fields,
-    //     taxonomy: source.source_taxonomy,
-    //     type: source.source_type
-    //   },
-
-
-    titles_to_remove = ['See also', 'References', 'Further reading', 'External links', 'Notes and references', 'Bibliography', 'Notes', 'Cited sources'];
-    block_types_toremove = ['table','empty_line'];
-    constructor(sourceContent:any){
-        this.content = sourceContent;
+export class ParseSourceContent {
+  public content: any;
+  /// Format of Content
+  // content: {
+  //     title: source.title,
+  //     headings: source.headings,
+  //     content: source.content,
+  //     fields: fields,
+  //     taxonomy: source.source_taxonomy,
+  //     type: source.source_type
+  //   },
+
+  titles_to_remove = [
+    "See also",
+    "References",
+    "Further reading",
+    "External links",
+    "Notes and references",
+    "Bibliography",
+    "Notes",
+    "Cited sources",
+  ];
+  block_types_toremove = ["table", "empty_line"];
+  constructor(sourceContent: any) {
+    this.content = sourceContent;
+  }
+
+  parseData() {
+    let sourceType = this.content.type;
+    let afterSanitized;
+    if (sourceType == "video") {
+      afterSanitized = this.parseVideoContent(this.content.content);
+    } else {
+      let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(
+        this.content.content
+      );
+      afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
     }
-
-    parseData() {
-            let sourceType = this.content.type;
-            let afterSanitized;
-            if(sourceType == "video"){
-                afterSanitized = this.parseVideoContent(this.content.content);
-            }else{
-            let dataAfterRemovingUnWantedBlocks = this.removeSectionsByTitle(this.content.content);
-            afterSanitized = this.sanitizeBlocks(dataAfterRemovingUnWantedBlocks);
-        }
-            return {
-                type: this.content.type,
-                title: this.content.title,
-                content: afterSanitized,
-                headings: this.content.headings,
-                taxonomy: this.content.taxonomy,
-            }    
+    return {
+      type: this.content.type,
+      title: this.content.title,
+      content: afterSanitized,
+      headings: this.content.headings,
+      taxonomy: this.content.taxonomy,
+    };
+  }
+
+  removeSectionsByTitle(data: Array<any>) {
+    let dataAfterRemoving = [];
+    for (let elem of data) {
+      if (
+        elem.block_type == "heading" &&
+        this.titles_to_remove.includes(elem.content)
+      ) {
+        continue;
+      }
+      /// remove unwanted blcok types , for now `table` and `empty_line`
+      if (this.block_types_toremove.includes(elem.block_type)) {
+        continue;
+      }
+      if (elem.children) {
+        elem.children = this.removeSectionsByTitle(elem.children);
+      }
+      dataAfterRemoving.push(elem);
     }
-
-    removeSectionsByTitle(data: Array<any>){
-        let dataAfterRemoving = [];
-        for(let elem of data){
-            if(elem.block_type == 'heading' &&   this.titles_to_remove.includes(elem.content)){
-                continue;
-            }
-            /// remove unwanted blcok types , for now `table` and `empty_line`
-            if(this.block_types_toremove.includes(elem.block_type)){
-                continue;
-            }
-            if(elem.children){
-                elem.children = this.removeSectionsByTitle(elem.children)
-                
-            }
-            dataAfterRemoving.push(elem)
-            
+    return dataAfterRemoving;
+  }
+
+  sanitizeTextContent(content: String) {
+    // Remove newline characters
+    content = content.replace(/\\n/g, " ");
+
+    // Remove internal link references, keeping only the link text
+    // Pattern explanation: [[link|text|index|wiki]] --> text
+    content = content.replace(/\[\[.*?\|(.*?)\|.*?\|wiki\]\]/g, "$1");
+
+    // Remove external links, keeping only the link text
+    // Pattern explanation: [url text] --> text
+    content = content.replace(/\[http[s]?:\/\/[^\s]+ ([^\]]+)\]/g, "$1");
+
+    // Remove Markdown link references, keeping only the link text
+    // Pattern explanation: ![link text](url) --> link text
+    content = content.replace(/\!\[([^\]]+)\]\([^\)]+\)/g, "$1");
+
+    return content;
+  }
+
+  sanitizeBlocks(blocks: Array<any>) {
+    let sanitizedBlocks = <any>[];
+    blocks = blocks.filter((item) => item.block_type != "table");
+    blocks.forEach((block) => {
+      let sanitizedBlock: any = {};
+      for (let key in block) {
+        let value = block[key];
+        if (typeof value === "string") {
+          sanitizedBlock[key] = this.sanitizeTextContent(value);
+        } else if (Array.isArray(value)) {
+          sanitizedBlock[key] = this.sanitizeBlocks(value);
+        } else {
+          sanitizedBlock[key] = value;
         }
-        return dataAfterRemoving;
-    }
-    
-    sanitizeTextContent(content: String) {
-        // Remove newline characters
-        content = content.replace(/\\n/g, ' ');
-    
-        // Remove internal link references, keeping only the link text
-        // Pattern explanation: [[link|text|index|wiki]] --> text
-        content = content.replace(/\[\[.*?\|(.*?)\|.*?\|wiki\]\]/g, '$1');
-    
-        // Remove external links, keeping only the link text
-        // Pattern explanation: [url text] --> text
-        content = content.replace(/\[http[s]?:\/\/[^\s]+ ([^\]]+)\]/g, '$1');
-    
-        // Remove Markdown link references, keeping only the link text
-        // Pattern explanation: ![link text](url) --> link text
-        content = content.replace(/\!\[([^\]]+)\]\([^\)]+\)/g, '$1');
-    
-        return content;
-    }
-
-    sanitizeBlocks(blocks: Array<any>) {
-        let sanitizedBlocks = <any>[] ;
-        blocks = blocks.filter((item)=> item.block_type != 'table');
-        blocks.forEach(block => {
-
-            let sanitizedBlock: any = {};
-            for (let key in block) {
-                let value = block[key];
-                if (typeof value === 'string') {
-                    sanitizedBlock[key] = this.sanitizeTextContent(value);
-                } else if (Array.isArray(value)) {
-                    sanitizedBlock[key] = this.sanitizeBlocks(value);
-                } else {
-                    sanitizedBlock[key] = value;
-                }
-            }
-            sanitizedBlocks.push(sanitizedBlock);
-        });
-        return sanitizedBlocks;
-    }
+      }
+      sanitizedBlocks.push(sanitizedBlock);
+    });
+    return sanitizedBlocks;
+  }
+
+  parseVideoContent(data: Array<any>) {
+    let finalChapters: Array<any> = [];
+    // let cleanedData = this.cleanTranscript(timeCodes);
+    data.forEach((e) => {
+      let combinedContent = this.cleanTranscript(e);
+      finalChapters.push({
+        startTime: e.startTime,
+        endTime: e.endTime,
+        content: combinedContent,
+        title: e.content,
+      });
+    });
 
-    parseVideoContent(data: Array<any>){
-        let finalChapters :Array<any> = [];
-        // let cleanedData = this.cleanTranscript(timeCodes);
-        data.forEach((e)=>{
-            let combinedContent = this.cleanTranscript(e);
-            finalChapters.push({
-                "startTime": e.startTime,
-                "endTime": e.endTime,
-                "content": combinedContent,
-                "title": e.content
-            });
-        });
-
-        return finalChapters;
-        
-        
-    }
+    return finalChapters;
+  }
 
-    // remove content inside [] which denotes non-speech sounds
- isNonSpeech(content: string) {
+  // remove content inside [] which denotes non-speech sounds
+  isNonSpeech(content: string) {
     // Check if the content is non-speech (enclosed in square brackets).
     return /^\[.*\]$/.test(content.trim());
-}
+  }
 
-// remove non-essential content
- cleanTranscript(data: any) {
-    let finalContent = '';
+  // remove non-essential content
+  cleanTranscript(data: any) {
+    let finalContent = "";
     let children = data.children ?? [];
 
-    children.forEach((e:any)=>{
-        let content = (e.content || "").trim();
+    children.forEach((e: any) => {
+      let content = (e.content || "").trim();
 
-        if(this.isNonSpeech(content)) return;
+      if (this.isNonSpeech(content)) return;
 
-        content = content.replace(/\s+/g, ' ');
-        finalContent += content;
+      content = content.replace(/\s+/g, " ");
+      finalContent += content;
     });
 
     return finalContent;
-}
+  }
 
-// collapse the timecode to 30 seconds
+  // collapse the timecode to 30 seconds
   collapseTimeCodes(data: Array<any>, maxDuration = 30.0) {
     // Collapse time codes into buckets of approximately maxDuration seconds.
     const collapsedData = [];
-    let  bucketStartTime: number | null = null;
-    let bucketEndTime : number | null = null;
-    let bucketContent : Array<any> = [];
+    let bucketStartTime: number | null = null;
+    let bucketEndTime: number | null = null;
+    let bucketContent: Array<any> = [];
     let bucketDuration = 0.0;
 
-    data.forEach(entry => {
-        const startTime = entry.start_time;
-        const endTime = entry.end_time;
-        const content = entry.content;
-        const entryDuration = endTime - startTime;
-
-        if (bucketStartTime === null) {
-            // Start a new bucket
-            bucketStartTime = startTime;
-            bucketEndTime = endTime;
-            bucketContent.push(content);
-            bucketDuration = entryDuration;
-        } else if ((bucketDuration + entryDuration) <= maxDuration) {
-            // Add to current bucket
-            bucketEndTime = endTime;
-            bucketContent.push(content);
-            bucketDuration += entryDuration;
-        } else {
-            // Close current bucket and start a new one
-            const collapsedEntry = {
-                start_time: bucketStartTime,
-                end_time: bucketEndTime,
-                content: bucketContent.join(' ')
-            };
-            collapsedData.push(collapsedEntry);
-
-            // Start new bucket with current entry
-            bucketStartTime = startTime;
-            bucketEndTime = endTime;
-            bucketContent = [content];
-            bucketDuration = entryDuration;
-        }
+    data.forEach((entry) => {
+      const startTime = entry.start_time;
+      const endTime = entry.end_time;
+      const content = entry.content;
+      const entryDuration = endTime - startTime;
+
+      if (bucketStartTime === null) {
+        // Start a new bucket
+        bucketStartTime = startTime;
+        bucketEndTime = endTime;
+        bucketContent.push(content);
+        bucketDuration = entryDuration;
+      } else if (bucketDuration + entryDuration <= maxDuration) {
+        // Add to current bucket
+        bucketEndTime = endTime;
+        bucketContent.push(content);
+        bucketDuration += entryDuration;
+      } else {
+        // Close current bucket and start a new one
+        const collapsedEntry = {
+          start_time: bucketStartTime,
+          end_time: bucketEndTime,
+          content: bucketContent.join(" "),
+        };
+        collapsedData.push(collapsedEntry);
+
+        // Start new bucket with current entry
+        bucketStartTime = startTime;
+        bucketEndTime = endTime;
+        bucketContent = [content];
+        bucketDuration = entryDuration;
+      }
     });
 
     // Add the last bucket if it exists
     if (bucketContent.length > 0) {
-        const collapsedEntry = {
-            start_time: bucketStartTime,
-            end_time: bucketEndTime,
-            content: bucketContent.join(' ')
-        };
-        collapsedData.push(collapsedEntry);
+      const collapsedEntry = {
+        start_time: bucketStartTime,
+        end_time: bucketEndTime,
+        content: bucketContent.join(" "),
+      };
+      collapsedData.push(collapsedEntry);
     }
 
     return collapsedData;
+  }
 }
-
-    
-    
-}

package/src/parse/response_format_card.ts

@@ -384,5 +384,3 @@ const sourceResp = {
   generated_at: 1718625081,
   type: "card_gen",
 };
-
-

package/src/parse/response_format_typology.ts

@@ -1,44 +1,44 @@
 const typologyResponse = {
-    "usage_data": {
-      "prompt_tokens": 11611,
-      "completion_tokens": 441,
-      "total_tokens": 12052
-    },
-    "generated_content": {
-      "field": [
-        "Sciences",
-        "Technology & Engineering",
-        "Education, Learning & Personal Development"
-      ],
-      "concepts": [
-        "Electrolysis",
-        "Faraday's Laws of Electrolysis",
-        "Electrolytic Cell",
-        "Decomposition Potential",
-        "Oxidation and Reduction at the Electrodes",
-        "Electrolysis of Water",
-        "Electrolysis of Carbon Dioxide",
-        "Electrocrystallization"
-      ],
-      "facts": [
-        "Electrolysis is the passing of a direct electric current through an electrolyte producing chemical reactions at the electrodes and decomposition of the materials.",
-        "In electrolysis, the quantity of the products is proportional to the current, and when two or more electrolytic cells are connected in series to the same power source, the products produced in the cells are proportional to their equivalent weight.",
-        "The main components required to achieve electrolysis are an electrolyte, electrodes, and an external power source.",
-        "Faraday's laws of electrolysis detail the amount of the products of electrolysis is related to the number of electrons in the reaction at the electrodes.",
-        "Decomposition potential or decomposition voltage refers to the minimum voltage between anode and cathode of an electrolytic cell that is needed for electrolysis to occur.",
-        "The electrochemical reduction of carbon dioxide can produce value-added chemicals such as methane, ethylene, and ethanol."
-      ],
-      "generate_cards": true,
-      "summary_cards": [
-        "Electrolysis is the process of passing direct electric current through an electrolyte, resulting in chemical reactions and the decomposition of materials.",
-        "Faraday's laws of electrolysis determine the relationship between the amounts of products generated and the electrons involved in the reaction at the electrodes.",
-        "Decomposition potential is the minimum voltage required between anode and cathode for electrolysis to occur.",
-        "The electrochemical reduction of carbon dioxide is a potential method for producing valuable chemicals such as methane, ethylene, and ethanol."
-      ]
-    },
-    "generated_at": "Tue, 20 Jan 1970 21:17:46 GMT"
-  };
+  usage_data: {
+    prompt_tokens: 11611,
+    completion_tokens: 441,
+    total_tokens: 12052,
+  },
+  generated_content: {
+    field: [
+      "Sciences",
+      "Technology & Engineering",
+      "Education, Learning & Personal Development",
+    ],
+    concepts: [
+      "Electrolysis",
+      "Faraday's Laws of Electrolysis",
+      "Electrolytic Cell",
+      "Decomposition Potential",
+      "Oxidation and Reduction at the Electrodes",
+      "Electrolysis of Water",
+      "Electrolysis of Carbon Dioxide",
+      "Electrocrystallization",
+    ],
+    facts: [
+      "Electrolysis is the passing of a direct electric current through an electrolyte producing chemical reactions at the electrodes and decomposition of the materials.",
+      "In electrolysis, the quantity of the products is proportional to the current, and when two or more electrolytic cells are connected in series to the same power source, the products produced in the cells are proportional to their equivalent weight.",
+      "The main components required to achieve electrolysis are an electrolyte, electrodes, and an external power source.",
+      "Faraday's laws of electrolysis detail the amount of the products of electrolysis is related to the number of electrons in the reaction at the electrodes.",
+      "Decomposition potential or decomposition voltage refers to the minimum voltage between anode and cathode of an electrolytic cell that is needed for electrolysis to occur.",
+      "The electrochemical reduction of carbon dioxide can produce value-added chemicals such as methane, ethylene, and ethanol.",
+    ],
+    generate_cards: true,
+    summary_cards: [
+      "Electrolysis is the process of passing direct electric current through an electrolyte, resulting in chemical reactions and the decomposition of materials.",
+      "Faraday's laws of electrolysis determine the relationship between the amounts of products generated and the electrons involved in the reaction at the electrodes.",
+      "Decomposition potential is the minimum voltage required between anode and cathode for electrolysis to occur.",
+      "The electrochemical reduction of carbon dioxide is a potential method for producing valuable chemicals such as methane, ethylene, and ethanol.",
+    ],
+  },
+  generated_at: "Tue, 20 Jan 1970 21:17:46 GMT",
+};
 
-  export function returnTypologyData(){
-    return typologyResponse as any;
-  }
+export function returnTypologyData() {
+  return typologyResponse as any;
+}

package/src/services/open_ai_service.ts

@@ -1,54 +1,56 @@
 import axios from "axios";
-import { parseOpenAiFailureResponse, parseOpenAiSuccessResponse } from "../utils/parse_openai_response";
+import {
+  parseOpenAiFailureResponse,
+  parseOpenAiSuccessResponse,
+} from "../utils/parse_openai_response";
 import { openAiEndPoint } from "../constants/api_constants";
 
+export class OpenAiService {
+  public api_key: string;
+  public model: string;
 
-export class OpenAiService{
-    public api_key: string;
-    public model: string;
-    
-    constructor(apiKey: string, model:string){
-        this.api_key = apiKey;
-        this.model =model
-    }
+  constructor(apiKey: string, model: string) {
+    this.api_key = apiKey;
+    this.model = model;
+  }
 
-    async sendRequest(prompt: string,content: string){
-        try{
-            let message = [
-                {
-                    "role": "system", "content": prompt
-                },{
-                    "role": "user", "content": content
-                }
-            ];
-            const url = openAiEndPoint();
-            let response = await axios.post(
-                url,
-                { 
-                    "model": this.model,
-                    "messages": message,
-                    "response_format": {"type": "json_object"},
-                },
-                {
-                    headers : {
-                        Authorization : "Bearer "+ this.api_key,
-                        "Content-Type" : ['application/json']
-                    },
-                    
-                }
-        
-            );
-        
-            if(response.status == 200){
-                console.log('success');
-                return parseOpenAiSuccessResponse(response.data) as any
-        
-            }else{
-                console.log('failed');
-                return response.statusText as any;
-            }}catch (err:any) {
-                return parseOpenAiFailureResponse(err.response);
-            }
-    } 
-}
+  async sendRequest(prompt: string, content: string) {
+    try {
+      let message = [
+        {
+          role: "system",
+          content: prompt,
+        },
+        {
+          role: "user",
+          content: content,
+        },
+      ];
+      const url = openAiEndPoint();
+      let response = await axios.post(
+        url,
+        {
+          model: this.model,
+          messages: message,
+          response_format: { type: "json_object" },
+        },
+        {
+          headers: {
+            Authorization: "Bearer " + this.api_key,
+            "Content-Type": ["application/json"],
+          },
+        }
+      );
 
+      if (response.status == 200) {
+        console.log("success");
+        return parseOpenAiSuccessResponse(response.data) as any;
+      } else {
+        console.log("failed");
+        return response.statusText as any;
+      }
+    } catch (err: any) {
+      return parseOpenAiFailureResponse(err.response);
+    }
+  }
+}