maidr 1.2.2 → 1.3.1

package/dist/maidr.js

@@ -74,13 +74,16 @@ class Constants {
   visualBraille = false; // do we want to represent braille based on what's visually there or actually there. Like if we have 2 outliers with the same position, do we show 1 (visualBraille true) or 2 (false)
   globalMinMax = true;
   ariaMode = 'assertive'; // assertive (default) / polite
-  playLLMWaitingSound = true;
 
   // LLM settings
-  LLMDebugMode = 0; // 0 = use real data, 1 = all fake, 2 = real data but no image
-  authKey = null; // OpenAI authentication key, set in menu
+  openAIAuthKey = null; // OpenAI authentication key, set in menu
+  geminiAuthKey = null; // Gemini authentication key, set in menu
   LLMmaxResponseTokens = 1000; // max tokens to send to LLM, 20 for testing, 1000 ish for real
+  playLLMWaitingSound = true;
   LLMDetail = 'high'; // low (default for testing, like 100 tokens) / high (default for real, like 1000 tokens)
+  LLMModel = 'openai'; // openai (default) / gemini
+  LLMSystemMessage =
+    'You are a helpful assistant describing the chart to a blind person';
   skillLevel = 'basic'; // basic / intermediate / expert
   skillLevelOther = ''; // custom skill level
 
@@ -297,7 +300,7 @@ class Menu {
             <div class="modal-dialog" role="document" tabindex="0">
                 <div class="modal-content">
                     <div class="modal-header">
-                        <h4 class="modal-title">Menu</h4>
+                        <h2 class="modal-title">Menu</h2>
                         <button type="button" class="close" data-dismiss="modal" aria-label="Close">
                             <span aria-hidden="true">&times;</span>
                         </button>
@@ -406,7 +409,15 @@ class Menu {
                               }><label for="aria_mode_polite">Polite</label></p>
                               </fieldset></div>
                             <h5 class="modal-title">LLM Settings</h5>
-                            <p><input type="password" id="chatLLM_auth_key"> <label for="chatLLM_auth_key">OpenAI Authentication Key</label></p>
+                            <p>
+                                <select id="LLM_model">
+                                    <option value="openai">OpenAI Vision</option>
+                                    <option value="gemini">Gemini Pro Vision</option>
+                                </select>
+                                <label for="LLM_model">LLM Model</label>
+                            </p>
+                            <p id="openai_auth_key_container" class="hidden"><input type="password" id="openai_auth_key"> <label for="openai_auth_key">OpenAI Authentication Key</label></p>
+                            <p id="gemini_auth_key_container" class="hidden"><input type="password" id="gemini_auth_key"> <label for="gemini_auth_key">Gemini Authentication Key</label></p>
                             <p>
                                 <select id="skill_level">
                                     <option value="basic">Basic</option>
@@ -416,7 +427,7 @@ class Menu {
                                 </select>
                                 <label for="skill_level">Level of skill in statistical charts</label>
                             </p>
-                            <p id="skill_level_other_container" class="hidden"><input type="text" id="skill_level_other"> <label for="skill_level_other">"I have a(n) [X] understanding of statistical charts"</label></p>
+                            <p id="skill_level_other_container" class="hidden"><input type="text" placeholder="Very basic" id="skill_level_other"> <label for="skill_level_other">Describe your level of skill in statistical charts</label></p>
                         </div>
                     </div>
                     <div class="modal-footer">
@@ -483,6 +494,29 @@ class Menu {
       },
     ]);
 
+    // toggle auth key fields
+    constants.events.push([
+      document.getElementById('LLM_model'),
+      'change',
+      function (e) {
+        if (e.target.value == 'openai') {
+          document
+            .getElementById('openai_auth_key_container')
+            .classList.remove('hidden');
+          document
+            .getElementById('gemini_auth_key_container')
+            .classList.add('hidden');
+        } else if (e.target.value == 'gemini') {
+          document
+            .getElementById('openai_auth_key_container')
+            .classList.add('hidden');
+          document
+            .getElementById('gemini_auth_key_container')
+            .classList.remove('hidden');
+        }
+      },
+    ]);
+
     // Skill level other events
     constants.events.push([
       document.getElementById('skill_level'),
@@ -560,7 +594,6 @@ class Menu {
    */
   PopulateData() {
     document.getElementById('vol').value = constants.vol;
-    //document.getElementById('show_rect').checked = constants.showRect;
     document.getElementById('autoplay_rate').value = constants.autoPlayRate;
     document.getElementById('braille_display_length').value =
       constants.brailleDisplayLength;
@@ -569,14 +602,20 @@ class Menu {
     document.getElementById('max_freq').value = constants.MAX_FREQUENCY;
     document.getElementById('keypress_interval').value =
       constants.keypressInterval;
-    if (typeof constants.authKey == 'string') {
-      document.getElementById('chatLLM_auth_key').value = constants.authKey;
+    if (typeof constants.openAIAuthKey == 'string') {
+      document.getElementById('openai_auth_key').value =
+        constants.openAIAuthKey;
+    }
+    if (typeof constants.geminiAuthKey == 'string') {
+      document.getElementById('gemini_auth_key').value =
+        constants.geminiAuthKey;
     }
     document.getElementById('skill_level').value = constants.skillLevel;
     if (constants.skillLevelOther) {
       document.getElementById('skill_level_other').value =
         constants.skillLevelOther;
     }
+    document.getElementById('LLM_model').value = constants.LLMModel;
 
     // aria mode
     if (constants.ariaMode == 'assertive') {
@@ -586,6 +625,22 @@ class Menu {
       document.getElementById('aria_mode_polite').checked = true;
       document.getElementById('aria_mode_assertive').checked = false;
     }
+    // hide either openai or gemini auth key field
+    if (constants.LLMModel == 'openai') {
+      document
+        .getElementById('openai_auth_key_container')
+        .classList.remove('hidden');
+      document
+        .getElementById('gemini_auth_key_container')
+        .classList.add('hidden');
+    } else if (constants.LLMModel == 'gemini') {
+      document
+        .getElementById('openai_auth_key_container')
+        .classList.add('hidden');
+      document
+        .getElementById('gemini_auth_key_container')
+        .classList.remove('hidden');
+    }
     // skill level other
     if (constants.skillLevel == 'other') {
       document
@@ -599,7 +654,6 @@ class Menu {
    */
   SaveData() {
     constants.vol = document.getElementById('vol').value;
-    //constants.showRect = document.getElementById('show_rect').checked;
     constants.autoPlayRate = document.getElementById('autoplay_rate').value;
     constants.brailleDisplayLength = document.getElementById(
       'braille_display_length'
@@ -609,10 +663,12 @@ class Menu {
     constants.MAX_FREQUENCY = document.getElementById('max_freq').value;
     constants.keypressInterval =
       document.getElementById('keypress_interval').value;
-    constants.authKey = document.getElementById('chatLLM_auth_key').value;
+    constants.openAIAuthKey = document.getElementById('openai_auth_key').value;
+    constants.geminiAuthKey = document.getElementById('gemini_auth_key').value;
     constants.skillLevel = document.getElementById('skill_level').value;
     constants.skillLevelOther =
       document.getElementById('skill_level_other').value;
+    constants.LLMModel = document.getElementById('LLM_model').value;
 
     // aria
     if (document.getElementById('aria_mode_assertive').checked) {
@@ -650,7 +706,6 @@ class Menu {
   SaveDataToLocalStorage() {
     let data = {};
     data.vol = constants.vol;
-    //data.showRect = constants.showRect;
     data.autoPlayRate = constants.autoPlayRate;
     data.brailleDisplayLength = constants.brailleDisplayLength;
     data.colorSelected = constants.colorSelected;
@@ -658,9 +713,11 @@ class Menu {
     data.MAX_FREQUENCY = constants.MAX_FREQUENCY;
     data.keypressInterval = constants.keypressInterval;
     data.ariaMode = constants.ariaMode;
-    data.authKey = constants.authKey;
+    data.openAIAuthKey = constants.openAIAuthKey;
+    data.geminiAuthKey = constants.geminiAuthKey;
     data.skillLevel = constants.skillLevel;
     data.skillLevelOther = constants.skillLevelOther;
+    data.LLMModel = constants.LLMModel;
     localStorage.setItem('settings_data', JSON.stringify(data));
   }
   /**
@@ -670,7 +727,6 @@ class Menu {
     let data = JSON.parse(localStorage.getItem('settings_data'));
     if (data) {
       constants.vol = data.vol;
-      //constants.showRect = data.showRect;
       constants.autoPlayRate = data.autoPlayRate;
       constants.brailleDisplayLength = data.brailleDisplayLength;
       constants.colorSelected = data.colorSelected;
@@ -678,9 +734,11 @@ class Menu {
       constants.MAX_FREQUENCY = data.MAX_FREQUENCY;
       constants.keypressInterval = data.keypressInterval;
       constants.ariaMode = data.ariaMode;
-      constants.authKey = data.authKey;
+      constants.openAIAuthKey = data.openAIAuthKey;
+      constants.geminiAuthKey = data.geminiAuthKey;
       constants.skillLevel = data.skillLevel;
       constants.skillLevelOther = data.skillLevelOther;
+      constants.LLMModel = data.LLMModel ? data.LLMModel : constants.LLMModel;
     }
     this.PopulateData();
     this.UpdateHtml();
@@ -709,7 +767,7 @@ class ChatLLM {
             <div class="modal-dialog" role="document" tabindex="0">
                 <div class="modal-content">
                     <div class="modal-header">
-                        <h4 id="chatLLM_title" class="modal-title">Ask a Question</h4>
+                        <h2 id="chatLLM_title" class="modal-title">Ask a Question</h2>
                         <button type="button" class="close" data-dismiss="modal" aria-label="Close">
                             <span aria-hidden="true">&times;</span>
                         </button>
@@ -769,7 +827,7 @@ class ChatLLM {
       document,
       'keyup',
       function (e) {
-        if (e.key == '?') {
+        if (e.key == '?' && (e.ctrlKey || e.metaKey)) {
           chatLLM.Toggle(true);
         }
       },
@@ -824,43 +882,15 @@ class ChatLLM {
    * @returns {void}
    */
   Submit(text, img = null) {
-    // send text to LLM
-    let url = 'https://api.openai.com/v1/chat/completions';
-    //let url = 'temp';
-
-    let requestJson = this.GetLLMJRequestJson(text, img);
-    console.log(requestJson);
-
-    let xhr = new XMLHttpRequest();
-
     // start waiting sound
     if (constants.playLLMWaitingSound) {
       chatLLM.WaitingSound(true);
     }
 
-    if (constants.LLMDebugMode == 1) {
-      // do the below with a 5 sec delay
-      setTimeout(function () {
-        chatLLM.ProcessLLMResponse(chatLLM.fakeLLMResponseData());
-      }, 5000);
-    } else {
-      fetch(url, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer ' + constants.authKey,
-        },
-        body: JSON.stringify(requestJson),
-      })
-        .then((response) => response.json())
-        .then((data) => {
-          chatLLM.ProcessLLMResponse(data);
-        })
-        .catch((error) => {
-          chatLLM.WaitingSound(false);
-          console.error('Error:', error);
-          // also todo: handle errors somehow
-        });
+    if (constants.LLMModel == 'gemini') {
+      chatLLM.GeminiPrompt(text, img);
+    } else if (constants.LLMModel == 'openai') {
+      chatLLM.OpenAIPrompt(text, img);
     }
   }
 
@@ -910,8 +940,38 @@ class ChatLLM {
   ProcessLLMResponse(data) {
     chatLLM.WaitingSound(false);
     console.log('LLM response: ', data);
-    let text = data.choices[0].message.content;
-    chatLLM.DisplayChatMessage('LLM', text);
+    let text = '';
+    let LLMName = '';
+
+    if (constants.LLMModel == 'openai') {
+      LLMName = 'OpenAI';
+      text = data.choices[0].message.content;
+      let i = this.requestJson.messages.length;
+      this.requestJson.messages[i] = {};
+      this.requestJson.messages[i].role = 'assistant';
+      this.requestJson.messages[i].content = text;
+
+      if (data.error) {
+        chatLLM.DisplayChatMessage(LLMName, 'Error processing request.');
+      } else {
+        chatLLM.DisplayChatMessage(LLMName, text);
+      }
+    } else if (constants.LLMModel == 'gemini') {
+      LLMName = 'Gemini';
+      if (data.text()) {
+        text = data.text();
+        chatLLM.DisplayChatMessage(LLMName, text);
+      } else {
+        if (!data.error) {
+          data.error = 'Error processing request.';
+        }
+      }
+      if (data.error) {
+        chatLLM.DisplayChatMessage(LLMName, 'Error processing request.');
+      } else {
+        // todo: display actual response
+      }
+    }
   }
 
   /**
@@ -976,32 +1036,60 @@ class ChatLLM {
   /**
    * Gets running prompt info, appends the latest request, and packages it into a JSON object for the LLM.
    * @function
-   * @name GetLLMJRequestJson
+   * @name OpenAIPrompt
    * @memberof module:constants
    * @returns {json}
    */
-  GetLLMJRequestJson(text, img) {
+  OpenAIPrompt(text, img) {
+    // request init
+    let url = 'https://api.openai.com/v1/chat/completions';
+    let auth = constants.openAIAuthKey;
+    let requestJson = chatLLM.OpenAIJson(text, img);
+    console.log('LLM request: ', requestJson);
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: 'Bearer ' + auth,
+      },
+      body: JSON.stringify(requestJson),
+    })
+      .then((response) => response.json())
+      .then((data) => {
+        chatLLM.ProcessLLMResponse(data);
+      })
+      .catch((error) => {
+        chatLLM.WaitingSound(false);
+        console.error('Error:', error);
+        chatLLM.DisplayChatMessage('LLM', 'Error processing request.');
+        // also todo: handle errors somehow
+      });
+  }
+  OpenAIJson(text, img) {
+    let sysMessage = constants.LLMSystemMessage;
+    let backupMessage =
+      'Describe ' + singleMaidr.type + ' charts to a blind person';
+    // headers and sys message
     if (!this.requestJson) {
       this.requestJson = {};
       this.requestJson.model = 'gpt-4-vision-preview';
       this.requestJson.max_tokens = constants.LLMmaxResponseTokens; // note: if this is too short (tested with less than 200), the response gets cut off
-      //this.requestJson.detail = constants.LLMDetail;
+
+      // sys message
       this.requestJson.messages = [];
       this.requestJson.messages[0] = {};
       this.requestJson.messages[0].role = 'system';
-      this.requestJson.messages[0].content =
-        'You are a helpful assistant describing the chart to a blind person';
+      this.requestJson.messages[0].content = sysMessage;
     }
 
+    // user message
+    // if we have an image (first time only), send the image and the text, otherwise just the text
     let i = this.requestJson.messages.length;
     this.requestJson.messages[i] = {};
     this.requestJson.messages[i].role = 'user';
-    if (constants.LLMDebugMode == 2) {
-      // test message only, no image
-      this.requestJson.messages[i].content =
-        'Describe bar charts to a blind person';
-    } else if (img) {
-      let image_url = img;
+    if (img) {
+      // first message, include the img
       this.requestJson.messages[i].content = [
         {
           type: 'text',
@@ -1009,7 +1097,7 @@ class ChatLLM {
         },
         {
           type: 'image_url',
-          image_url: { url: image_url },
+          image_url: { url: img },
         },
       ];
     } else {
@@ -1020,6 +1108,47 @@ class ChatLLM {
     return this.requestJson;
   }
 
+  // Assuming this function is part of your existing JavaScript file
+  async GeminiPrompt(text, imgBase64 = null) {
+    try {
+      // Save the image for next time
+      if (imgBase64 == null) {
+        imgBase64 = constants.LLMImage;
+      } else {
+        constants.LLMImage = imgBase64;
+      }
+      constants.LLMImage = imgBase64;
+
+      // Import the module
+      const { GoogleGenerativeAI } = await import(
+        'https://esm.run/@google/generative-ai'
+      );
+      const API_KEY = constants.geminiAuthKey;
+      const genAI = new GoogleGenerativeAI(API_KEY);
+      const model = genAI.getGenerativeModel({ model: 'gemini-pro-vision' });
+
+      // Create the prompt
+      const prompt = constants.LLMSystemMessage + '\n\n' + text; // Use the text parameter as the prompt
+      const image = {
+        inlineData: {
+          data: imgBase64, // Use the base64 image string
+          mimeType: 'image/png', // Or the appropriate mime type of your image
+        },
+      };
+
+      // Generate the content
+      console.log('LLM request: ', prompt, image);
+      const result = await model.generateContent([prompt, image]);
+      console.log(result.response.text());
+
+      // Process the response
+      chatLLM.ProcessLLMResponse(result.response);
+    } catch (error) {
+      console.error('Error in GeminiPrompt:', error);
+      throw error; // Rethrow the error for further handling if necessary
+    }
+  }
+
   /**
    * Displays chat message from the user and LLM in a chat history window
    * @function
@@ -1032,7 +1161,7 @@ class ChatLLM {
       <div class="chatLLM_message ${
         user == 'User' ? 'chatLLM_message_self' : 'chatLLM_message_other'
       }">
-        <p class="chatLLM_message_user">${user}</p>
+        <h3 class="chatLLM_message_user">${user}</h3>
         <p class="chatLLM_message_text">${text}</p>
       </div>
     `;
@@ -1089,8 +1218,9 @@ class ChatLLM {
 
       // first time, send default query
       if (this.firstTime) {
+        let LLMName = constants.LLMModel == 'openai' ? 'OpenAI' : 'Gemini';
         this.firstTime = false;
-        this.DisplayChatMessage('LLM', 'Processing Chart...');
+        this.DisplayChatMessage(LLMName, 'Processing Chart...');
         this.RunDefaultPrompt();
       }
     } else {
@@ -1109,36 +1239,30 @@ class ChatLLM {
   async ConvertSVGtoJPG(id) {
     let svgElement = document.getElementById(id);
     return new Promise((resolve, reject) => {
-      // Create a canvas
       var canvas = document.createElement('canvas');
       var ctx = canvas.getContext('2d');
 
-      // Get dimensions from the SVG element
-      var svgRect = svgElement.getBoundingClientRect();
-      canvas.width = svgRect.width;
-      canvas.height = svgRect.height;
-
-      // Create an image to draw the SVG
-      var img = new Image();
-
-      // Convert SVG element to a data URL
       var svgData = new XMLSerializer().serializeToString(svgElement);
-      var svgBlob = new Blob([svgData], {
-        type: 'image/svg+xml;charset=utf-8',
-      });
-      var url = URL.createObjectURL(svgBlob);
-
-      img.onload = function () {
-        // Draw the SVG on the canvas
-        ctx.drawImage(img, 0, 0, svgRect.width, svgRect.height);
-
-        // Convert the canvas to JPEG
-        var jpegData = canvas.toDataURL('image/jpeg');
+      if (!svgData.startsWith('<svg xmlns')) {
+        svgData = `<svg xmlns="http://www.w3.org/2000/svg" ${svgData.slice(4)}`;
+      }
 
-        // Resolve the promise with the Base64 JPEG data
-        resolve(jpegData);
+      var svgSize =
+        svgElement.viewBox.baseVal || svgElement.getBoundingClientRect();
+      canvas.width = svgSize.width;
+      canvas.height = svgSize.height;
 
-        // Clean up
+      var img = new Image();
+      img.onload = function () {
+        ctx.drawImage(img, 0, 0, svgSize.width, svgSize.height);
+        var jpegData = canvas.toDataURL('image/jpeg', 0.9); // 0.9 is the quality parameter
+        if (constants.LLMModel == 'openai') {
+          resolve(jpegData);
+        } else if (constants.LLMModel == 'gemini') {
+          let base64Data = jpegData.split(',')[1];
+          resolve(base64Data);
+          //resolve(jpegData);
+        }
         URL.revokeObjectURL(url);
       };
 
@@ -1146,30 +1270,14 @@ class ChatLLM {
         reject(new Error('Error loading SVG'));
       };
 
+      var svgBlob = new Blob([svgData], {
+        type: 'image/svg+xml;charset=utf-8',
+      });
+      var url = URL.createObjectURL(svgBlob);
       img.src = url;
     });
   }
 
-  downloadJPEG(base64Data, filename) {
-    // Create a link element
-    var link = document.createElement('a');
-
-    // Set the download attribute with a filename
-    link.download = filename;
-
-    // Convert Base64 data to a data URL and set it as the href
-    link.href = base64Data;
-
-    // Append the link to the body (required for Firefox)
-    document.body.appendChild(link);
-
-    // Trigger the download
-    link.click();
-
-    // Clean up
-    document.body.removeChild(link);
-  }
-
   /**
    * RunDefaultPrompt is an asynchronous function that generates a prompt for describing a chart to a blind person.
    * It converts the chart to a JPG image using the ConvertSVGtoJPG method and then submits the prompt to the chatLLM function.
@@ -1178,7 +1286,6 @@ class ChatLLM {
   async RunDefaultPrompt() {
     //let img = await this.ConvertSVGtoImg(singleMaidr.id);
     let img = await this.ConvertSVGtoJPG(singleMaidr.id);
-    //this.downloadJPEG(img, 'test.jpg'); // test download
     let text = 'Describe this chart to a blind person';
     if (constants.skillLevel) {
       if (constants.skillLevel == 'other' && constants.skillLevelOther) {
@@ -1230,7 +1337,7 @@ class Description {
             <div class="modal-dialog" role="document" tabindex="0">
                 <div class="modal-content">
                     <div class="modal-header">
-                        <h4 id="desc_title" class="modal-title">Description</h4>
+                        <h2 id="desc_title" class="modal-title">Description</h2>
                         <button type="button" class="close" data-dismiss="modal" aria-label="Close">
                             <span aria-hidden="true">&times;</span>
                         </button>
@@ -2022,7 +2129,7 @@ class Audio {
         panning = 0;
       }
     } else if (constants.chartType == 'heat') {
-      rawFreq = plot.values[position.y][position.x];
+      rawFreq = plot.data[position.y][position.x];
       rawPanning = position.x;
       frequency = this.SlideBetween(
         rawFreq,
@@ -2818,7 +2925,7 @@ class Display {
           plot.fill +
           ' is ';
         // if (constants.hasRect) {
-        verboseText += plot.plotData[2][position.y][position.x];
+        verboseText += plot.data[position.y][position.x];
         // }
       } else {
         verboseText +=
@@ -2833,7 +2940,7 @@ class Display {
           plot.fill +
           ' is ';
         // if (constants.hasRect) {
-        verboseText += plot.plotData[2][position.y][position.x];
+        verboseText += plot.data[position.y][position.x];
         // }
       }
       // terse and verbose alternate between columns and rows
@@ -2847,7 +2954,7 @@ class Display {
             '<p>' +
             plot.x_labels[position.x] +
             ', ' +
-            plot.plotData[2][position.y][position.x] +
+            plot.data[position.y][position.x] +
             '</p>\n';
         } else {
           // row navigation
@@ -2855,7 +2962,7 @@ class Display {
             '<p>' +
             plot.y_labels[position.y] +
             ', ' +
-            plot.plotData[2][position.y][position.x] +
+            plot.data[position.y][position.x] +
             '</p>\n';
         }
       } else if (constants.textMode == 'verbose') {
@@ -3032,11 +3139,11 @@ class Display {
     } else if (constants.chartType == 'line') {
       // line layer
       verboseText +=
-        plot.x_group_label +
+        plot.plotLegend.x +
         ' is ' +
         plot.pointValuesX[position.x] +
         ', ' +
-        plot.y_group_label +
+        plot.plotLegend.y +
         ' is ' +
         plot.pointValuesY[position.x];
 
@@ -3709,6 +3816,8 @@ class BarChart {
     let elements = null;
     if ('selector' in singleMaidr) {
       elements = document.querySelectorAll(singleMaidr.selector);
+    } else if ('elements' in singleMaidr) {
+      elements = singleMaidr.elements;
     }
 
     if (xlevel && data && elements) {
@@ -3745,15 +3854,6 @@ class BarChart {
       logError.LogAbsentElement('elements');
     }
 
-    // bars. The actual bar elements in the SVG. Used to highlight visually
-    // if ('elements' in singleMaidr) {
-    //   this.bars = singleMaidr.elements;
-    //   constants.hasRect = 1;
-    // } else {
-    //   // this.bars = constants.chart.querySelectorAll('g[id^="geom_rect"] > rect'); // if we use plot.plotData.length instead of plot.bars.length, we don't have to include this
-    //   constants.hasRect = 0;
-    // }
-
     // column labels, both legend and tick
     this.columnLabels = [];
     let legendX = '';
@@ -3829,10 +3929,8 @@ class BarChart {
 
     if (Array.isArray(singleMaidr)) {
       this.plotData = singleMaidr;
-    } else {
-      if ('data' in singleMaidr) {
-        this.plotData = singleMaidr.data;
-      }
+    } else if ('data' in singleMaidr) {
+      this.plotData = singleMaidr.data;
     }
 
     // set the max and min values for the plot
@@ -4023,8 +4121,6 @@ class BoxPlot {
    * @constructor
    */
   constructor() {
-    constants.plotOrientation = 'horz'; // default
-
     // the default sections for all boxplots
     this.sections = [
       'lower_outlier',
@@ -4036,6 +4132,8 @@ class BoxPlot {
       'upper_outlier',
     ];
 
+    // set orientation
+    constants.plotOrientation = 'horz';
     if ('axes' in singleMaidr) {
       if ('x' in singleMaidr.axes) {
         if ('level' in singleMaidr.axes.x) {
@@ -4116,7 +4214,11 @@ class BoxPlot {
 
     // bounds data
     if ('selector' in singleMaidr) {
-      this.plotBounds = this.GetPlotBounds();
+      let elements = document.querySelector(singleMaidr.selector);
+      this.plotBounds = this.GetPlotBounds(elements);
+      constants.hasRect = true;
+    } else if ('elements' in singleMaidr) {
+      this.plotBounds = this.GetPlotBounds(singleMaidr.elements);
       constants.hasRect = true;
     } else {
       constants.hasRect = false;
@@ -4202,7 +4304,7 @@ class BoxPlot {
    * Calculates the bounding boxes for all elements in the parent element, including outliers, whiskers, and range.
    * @returns {Array} An array of bounding boxes for all elements.
    */
-  GetPlotBounds() {
+  GetPlotBounds(elements) {
     // we fetch the elements in our parent,
     // and similar to old GetData we run through and get bounding boxes (or blanks) for everything,
     // and store in an identical structure
@@ -4210,7 +4312,6 @@ class BoxPlot {
     let plotBounds = [];
     let allWeNeed = this.GetAllSegmentTypes();
     let re = /(?:\d+(?:\.\d*)?|\.\d+)/g;
-    let elements = document.querySelector(singleMaidr.selector);
 
     // get initial set of elements, a parent element for all outliers, whiskers, and range
     let initialElemSet = [];
@@ -4781,100 +4882,36 @@ class HeatMap {
         }
       }
     }
-    let data = null;
-    let dataLength = 0;
     if ('data' in singleMaidr) {
-      data = singleMaidr.data;
-      for (let i = 0; i < data.length; i++) {
-        dataLength += data[i].length;
-      }
+      this.data = singleMaidr.data;
+      this.num_rows = this.data.length;
+      this.num_cols = this.data[0].length;
+    } else {
+      // critical error, no data
+      console.error('No data found in singleMaidr object');
     }
-    let elements = null;
     if ('selector' in singleMaidr) {
-      elements = document.querySelectorAll(singleMaidr.selector);
+      this.elements = document.querySelectorAll(singleMaidr.selector);
+      constants.hasRect = 1;
+    } else if ('elements' in singleMaidr) {
+      this.elements = singleMaidr.elements;
+      constants.hasRect = 1;
+    } else {
+      this.elements = null;
+      constants.hasRect = 0;
     }
 
-    // if (xlevel && ylevel && data && elements) {
-    //   if (elements.length != dataLength) {
-    //     // I didn't throw an error but give a warning
-    //     constants.hasRect = 0;
-    //     logError.LogDifferentLengths('data', 'elements');
-    //   } else if (ylevel.length != data.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('y level', 'rows');
-    //   } else if (data[0].length != xlevel.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('x level', 'columns');
-    //   } else {
-    //     this.plots = elements;
-    //     constants.hasRect = 1;
-    //   }
-    // } else if (ylevel && data && elements) {
-    //   if (dataLength != elements.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('data', 'elements');
-    //   } else if (ylevel.length != data.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('y level', 'rows');
-    //   } else {
-    //     this.plots = elements;
-    //     constants.hasRect = 1;
-    //   }
-    // } else if (xlevel && data && elements) {
-    //   if (dataLength != elements.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('data', 'elements');
-    //   } else if (xlevel.length != data[0].length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('x level', 'columns');
-    //   } else {
-    //     this.plots = elements;
-    //     constants.hasRect = 1;
-    //   }
-    // }
-    // else if (xlevel && ylevel && data) {
-    //   constants.hasRect = 0;
-    //   if (ylevel.length != data.length) {
-    //     logError.logDifferentLengths('y level', 'rows');
-    //   } else if (data[0].length != xlevel.length) {
-    //     logError.logDifferentLengths('x level', 'columns');
-    //   }
-    //   logError.LogAbsentElement('elements');
-    // }
-    // else if (data && elements) {
-    //   if (dataLength != elements.length) {
-    //     constants.hasRect = 0;
-    //     logError.logDifferentLengths('data', 'elements');
-    //   } else {
-    //     this.plots = elements;
-    //     constants.hasRect = 1;
-    //   }
-    // } else if (data) {
-    //   constants.hasRect = 0;
-    //   if (!xlevel) logError.LogAbsentElement('x level');
-    //   if (!ylevel) logError.LogAbsentElement('y level');
-    //   if (!elements) logError.LogAbsentElement('elements');
-    // }
-
-    this.plots = elements;
-    constants.hasRect = 1;
-
     this.group_labels = this.getGroupLabels();
-    // this.x_labels = this.getXLabels();
-    // this.y_labels = this.getYLabels();
     this.x_labels = xlevel;
     this.y_labels = ylevel;
     this.title = this.getTitle();
     this.fill = this.getFill();
 
-    this.plotData = this.getHeatMapData();
-    this.updateConstants();
+    if (constants.hasRect) {
+      this.SetHeatmapRectData();
+    }
 
-    this.x_coord = this.plotData[0];
-    this.y_coord = this.plotData[1];
-    this.values = this.plotData[2];
-    this.num_rows = this.plotData[3];
-    this.num_cols = this.plotData[4];
+    this.updateConstants();
 
     this.x_group_label = this.group_labels[0].trim();
     this.y_group_label = this.group_labels[1].trim();
@@ -4885,111 +4922,77 @@ class HeatMap {
    * If 'data' exists in singleMaidr, it returns the norms from the data. Otherwise, it calculates the norms from the unique x and y coordinates.
    * @returns {Array} An array of heatmap data containing unique x and y coordinates, norms, number of rows, and number of columns.
    */
-  getHeatMapData() {
+  SetHeatmapRectData() {
+    // We get a set of x and y coordinates from the heatmap squares,
+    // which is different and only sometimes connected to the actual data
+    // note, only runs if constants.hasRect is true
+
     // get the x_coord and y_coord to check if a square exists at the coordinates
     let x_coord_check = [];
     let y_coord_check = [];
-
     let unique_x_coord = [];
     let unique_y_coord = [];
-    if (constants.hasRect) {
-      for (let i = 0; i < this.plots.length; i++) {
-        if (this.plots[i]) {
-          // heatmap SVG containing path element instead of rect
-          if (this.plots[i] instanceof SVGPathElement) {
-            // Assuming the path data is in the format "M x y L x y L x y L x y"
-            const path_d = this.plots[i].getAttribute('d');
-            const coords = path_d.match(/[\d\.]+/g).map(Number);
-
-            const x = coords[0];
-            const y = coords[1];
-
-            x_coord_check.push(parseFloat(x));
-            y_coord_check.push(parseFloat(y));
-          } else {
-            x_coord_check.push(parseFloat(this.plots[i].getAttribute('x')));
-            y_coord_check.push(parseFloat(this.plots[i].getAttribute('y')));
-          }
+    for (let i = 0; i < this.elements.length; i++) {
+      if (this.elements[i]) {
+        // heatmap SVG containing path element instead of rect
+        if (this.elements[i] instanceof SVGPathElement) {
+          // Assuming the path data is in the format "M x y L x y L x y L x y"
+          const path_d = this.elements[i].getAttribute('d');
+          const regex = /[ML]\s*(-?\d+(\.\d+)?)\s+(-?\d+(\.\d+)?)/g;
+          const match = regex.exec(path_d);
+
+          const coords = [Number(match[1]), Number(match[3])];
+          const x = coords[0];
+          const y = coords[1];
+
+          x_coord_check.push(parseFloat(x));
+          y_coord_check.push(parseFloat(y));
+        } else {
+          x_coord_check.push(parseFloat(this.elements[i].getAttribute('x')));
+          y_coord_check.push(parseFloat(this.elements[i].getAttribute('y')));
         }
       }
-
-      // sort the squares to access from left to right, up to down
-      x_coord_check.sort(function (a, b) {
-        return a - b;
-      }); // ascending
-      y_coord_check.sort(function (a, b) {
-        return a - b;
-      });
-
-      let svgScaler = this.GetSVGScaler();
-      // inverse scale if svg is scaled
-      if (svgScaler[0] == -1) {
-        x_coord_check = x_coord_check.reverse();
-      }
-      if (svgScaler[1] == -1) {
-        y_coord_check = y_coord_check.reverse();
-      }
-
-      // get unique elements from x_coord and y_coord
-      unique_x_coord = [...new Set(x_coord_check)];
-      unique_y_coord = [...new Set(y_coord_check)];
-    }
-
-    // get num of rows, num of cols, and total numbers of squares
-    let num_rows = 0;
-    let num_cols = 0;
-    let num_squares = 0;
-    if ('data' in singleMaidr) {
-      num_rows = singleMaidr.data.length;
-      num_cols = singleMaidr.data[0].length;
-    } else {
-      num_rows = unique_y_coord.length;
-      num_cols = unique_x_coord.length;
     }
-    num_squares = num_rows * num_cols;
 
-    let norms = [];
-    if ('data' in singleMaidr) {
-      norms = [...singleMaidr.data];
-    } else {
-      norms = Array(num_rows)
-        .fill()
-        .map(() => Array(num_cols).fill(0));
-      let min_norm = 3 * Math.pow(255, 2);
-      let max_norm = 0;
-
-      for (var i = 0; i < this.plots.length; i++) {
-        var x_index = unique_x_coord.indexOf(x_coord_check[i]);
-        var y_index = unique_y_coord.indexOf(y_coord_check[i]);
-        let norm = this.getRGBNorm(i);
-        norms[y_index][x_index] = norm;
+    // sort the squares to access from left to right, up to down
+    x_coord_check.sort(function (a, b) {
+      return a - b;
+    }); // ascending
+    y_coord_check.sort(function (a, b) {
+      return a - b;
+    });
 
-        if (norm < min_norm) min_norm = norm;
-        if (norm > max_norm) max_norm = norm;
-      }
+    let svgScaler = this.GetSVGScaler();
+    // inverse scale if svg has a negative scale in the actual svg
+    if (svgScaler[0] == -1) {
+      x_coord_check = x_coord_check.reverse();
+    }
+    if (svgScaler[1] == -1) {
+      y_coord_check = y_coord_check.reverse();
     }
 
-    let plotData = [unique_x_coord, unique_y_coord, norms, num_rows, num_cols];
+    // get unique elements from x_coord and y_coord
+    unique_x_coord = [...new Set(x_coord_check)];
+    unique_y_coord = [...new Set(y_coord_check)];
 
-    return plotData;
+    this.x_coord = unique_x_coord;
+    this.y_coord = unique_y_coord;
   }
 
   /**
    * Updates the constants used in the heatmap.
+   * minX: 0, always
+   * maxX: the x length of the data array
+   * minY: the minimum value of the data array
+   * maxY: the maximum value of the data array
+   * autoPlayRate: the rate at which the heatmap will autoplay, based on the number of columns
+   *
    */
   updateConstants() {
     constants.minX = 0;
-    constants.maxX = this.plotData[4];
-    constants.minY = this.plotData[2][0][0]; // initial val
-    constants.maxY = this.plotData[2][0][0]; // initial val
-    for (let i = 0; i < this.plotData[2].length; i++) {
-      for (let j = 0; j < this.plotData[2][i].length; j++) {
-        if (this.plotData[2][i][j] < constants.minY)
-          constants.minY = this.plotData[2][i][j];
-        if (this.plotData[2][i][j] > constants.maxY)
-          constants.maxY = this.plotData[2][i][j];
-      }
-    }
+    constants.maxX = this.data[0].length - 1;
+    constants.minY = Math.min(...this.data.map((row) => Math.min(...row)));
+    constants.maxY = Math.max(...this.data.map((row) => Math.max(...row)));
     constants.autoPlayRate = Math.min(
       Math.ceil(constants.AUTOPLAY_DURATION / (constants.maxX + 1)),
       constants.MAX_SPEED
@@ -5008,7 +5011,7 @@ class HeatMap {
   }
 
   /**
-   * Returns an array of the X and Y scales of the first SVG element found in the plots array.
+   * Returns an array of the X and Y scales of the first SVG element found in the elements array.
    * @returns {Array<number>} An array containing the X and Y scales of the SVG element.
    */
   GetSVGScaler() {
@@ -5018,7 +5021,7 @@ class HeatMap {
 
     // but first, are we even in an svg that can be scaled?
     let isSvg = false;
-    let element = this.plots[0]; // a random start, may as well be the first
+    let element = this.elements[0]; // a random start, may as well be the first
     while (element) {
       if (element.tagName.toLowerCase() == 'body') {
         break;
@@ -5030,7 +5033,7 @@ class HeatMap {
     }
 
     if (isSvg) {
-      let element = this.plots[0]; // a random start, may as well be the first
+      let element = this.elements[0]; // a random start, may as well be the first
       while (element) {
         if (element.tagName.toLowerCase() == 'body') {
           break;
@@ -5062,7 +5065,7 @@ class HeatMap {
    * @returns {number} The sum of squared values of the RGB color.
    */
   getRGBNorm(i) {
-    let rgb_string = this.plots[i].getAttribute('fill');
+    let rgb_string = this.elements[i].getAttribute('fill');
     let rgb_array = rgb_string.slice(4, -1).split(',');
     // just get the sum of squared value of rgb, similar without sqrt, save computation
     return rgb_array
@@ -5220,10 +5223,10 @@ class HeatMapRect {
     this.x = plot.x_coord[position.x];
     this.y = plot.y_coord[position.y];
     // find which square we're on by searching for the x and y coordinates
-    for (let i = 0; i < plot.plots.length; i++) {
+    for (let i = 0; i < plot.elements.length; i++) {
       if (
-        plot.plots[i].getAttribute('x') == this.x &&
-        plot.plots[i].getAttribute('y') == this.y
+        plot.elements[i].getAttribute('x') == this.x &&
+        plot.elements[i].getAttribute('y') == this.y
       ) {
         this.squareIndex = i;
         break;
@@ -5251,7 +5254,7 @@ class HeatMapRect {
     rect.setAttribute('stroke', constants.colorSelected);
     rect.setAttribute('stroke-width', this.rectStrokeWidth);
     rect.setAttribute('fill', 'none');
-    plot.plots[this.squareIndex].parentNode.appendChild(rect);
+    plot.elements[this.squareIndex].parentNode.appendChild(rect);
     //constants.chart.appendChild(rect);
   }
 }
@@ -5267,87 +5270,12 @@ class ScatterPlot {
    */
   constructor() {
     this.prefix = this.GetPrefix();
-    // this.SetVisualHighlight();
     this.SetScatterLayer();
     this.SetLineLayer();
     this.SetAxes();
     this.svgScaler = this.GetSVGScaler();
   }
 
-  // SetVisualHighlight() {
-  //   let point_index = this.GetElementIndex('point');
-  //   let smooth_index = this.GetElementIndex('smooth');
-  //   if (point_index && smooth_index && elements < 2) {
-  //     logError.LogAbsentElement('point or/and smooth line elements');
-  //   }
-  //   if (point_index != -1) {
-  //     this.CheckData(point_index);
-  //   }
-
-  //   if (smooth_index != -1) {
-  //     this.CheckData(smooth_index);
-  //   }
-  // }
-
-  // CheckData(i) {
-  //   let elements = 'elements' in singleMaidr ? singleMaidr.elements : null;
-
-  //   // elements does not exist at all
-  //   if (elements == null) {
-  //     logError.LogAbsentElement('elements');
-  //     if (i == 0) constants.hasRect = 0;
-  //     if (i == 1) constants.hasSmooth = 0;
-  //     return;
-  //   }
-
-  //   // elements exists but is empty
-  //   if (elements.length == 0) {
-  //     logError.LogAbsentElement('elements');
-  //     if (i == 0) constants.hasRect = 0;
-  //     if (i == 1) constants.hasSmooth = 0;
-  //     return;
-  //   }
-
-  //   // elements exists but is not an array
-  //   if (!Array.isArray(elements)) {
-  //     logError.LogNotArray('elements');
-  //     if (i == 0) constants.hasRect = 0;
-  //     if (i == 1) constants.hasSmooth = 0;
-  //     return;
-  //   }
-
-  //   // elements.length is more than 2
-  //   if (elements.length > 2) {
-  //     logError.LogTooManyElements('elements', 2);
-  //   }
-
-  //   if ('data' in singleMaidr) {
-  //     if (i == 0) {
-  //       // check point elements
-  //       if (
-  //         singleMaidr.data[i] == null ||
-  //         singleMaidr.data[i].length != singleMaidr.elements[i].length
-  //       ) {
-  //         constants.hasRect = 0;
-  //         logError.LogDifferentLengths('point data', 'point elements');
-  //       }
-  //     } else if (i == 1) {
-  //       // check smooth line elements
-  //       if (
-  //         singleMaidr.data[i] == null ||
-  //         (!Array.isArray(singleMaidr.data[i]) &&
-  //           singleMaidr.data[i].length != this.chartLineX.length)
-  //       ) {
-  //         constants.hasSmooth = 0;
-  //         logError.LogDifferentLengths(
-  //           'smooth line data',
-  //           'smooth line elements'
-  //         );
-  //       }
-  //     }
-  //   }
-  // }
-
   /**
    * Sets the x and y group labels and title for the scatterplot based on the data in singleMaidr.
    */
@@ -5393,28 +5321,34 @@ class ScatterPlot {
    */
   SetScatterLayer() {
     // initially set as smooth layer (layer 2), if possible
-    let elIndex = this.GetElementIndex('point');
+    let elIndex = this.GetElementIndex('point'); // check if we have it
     if (elIndex != -1) {
-      this.plotPoints = document.querySelectorAll(
-        singleMaidr.selector[elIndex]
-      );
+      if ('selector' in singleMaidr) {
+        this.plotPoints = document.querySelectorAll(
+          singleMaidr.selector[elIndex]
+        );
+      } else if ('elements' in singleMaidr) {
+        this.plotPoints = singleMaidr.elements[elIndex];
+      }
     } else if (singleMaidr.type == 'point') {
-      this.plotPoints = document.querySelectorAll(singleMaidr.selector);
+      if ('selector' in singleMaidr) {
+        this.plotPoints = document.querySelectorAll(singleMaidr.selector);
+      } else if ('elements' in singleMaidr) {
+        this.plotPoints = singleMaidr.elements;
+      }
     }
-    if (typeof this.plotPoints !== 'undefined') {
-      let svgPointCoords = this.GetSvgPointCoords();
-      let pointValues = this.GetPointValues();
+    let svgPointCoords = this.GetSvgPointCoords();
+    let pointValues = this.GetPointValues();
 
-      this.chartPointsX = svgPointCoords[0]; // x coordinates of points
-      this.chartPointsY = svgPointCoords[1]; // y coordinates of points
+    this.chartPointsX = svgPointCoords[0]; // x coordinates of points
+    this.chartPointsY = svgPointCoords[1]; // y coordinates of points
 
-      this.x = pointValues[0]; // actual values of x
-      this.y = pointValues[1]; // actual values of y
+    this.x = pointValues[0]; // actual values of x
+    this.y = pointValues[1]; // actual values of y
 
-      // for sound weight use
-      this.points_count = pointValues[2]; // number of each points
-      this.max_count = pointValues[3];
-    }
+    // for sound weight use
+    this.points_count = pointValues[2]; // number of each points
+    this.max_count = pointValues[3];
   }
 
   /**
@@ -5422,28 +5356,34 @@ class ScatterPlot {
    */
   SetLineLayer() {
     // layer = 2, smooth layer (from singleMaidr types)
-    let elIndex = this.GetElementIndex('smooth');
+    let elIndex = this.GetElementIndex('smooth'); // check if we have it
     if (elIndex != -1) {
-      this.plotLine = document.querySelectorAll(
-        singleMaidr.selector[elIndex]
-      )[0];
+      if ('selector' in singleMaidr) {
+        this.plotLine = document.querySelectorAll(
+          singleMaidr.selector[elIndex]
+        )[0];
+      } else if ('elements' in singleMaidr) {
+        this.plotLine = singleMaidr.elements[elIndex][0];
+      }
     } else if (singleMaidr.type == 'smooth') {
-      this.plotLine = document.querySelectorAll(singleMaidr.selector);
+      if ('selector' in singleMaidr) {
+        this.plotLine = document.querySelectorAll(singleMaidr.selector);
+      } else if ('elements' in singleMaidr) {
+        this.plotLine = singleMaidr.elements;
+      }
     }
-    if (typeof this.plotLine !== 'undefined') {
-      let svgLineCoords = this.GetSvgLineCoords();
-      let smoothCurvePoints = this.GetSmoothCurvePoints();
+    let svgLineCoords = this.GetSvgLineCoords();
+    let smoothCurvePoints = this.GetSmoothCurvePoints();
 
-      this.chartLineX = svgLineCoords[0]; // x coordinates of curve
-      this.chartLineY = svgLineCoords[1]; // y coordinates of curve
+    this.chartLineX = svgLineCoords[0]; // x coordinates of curve
+    this.chartLineY = svgLineCoords[1]; // y coordinates of curve
 
-      this.curveX = smoothCurvePoints[0]; // actual values of x
-      this.curvePoints = smoothCurvePoints[1]; // actual values of y
+    this.curveX = smoothCurvePoints[0]; // actual values of x
+    this.curvePoints = smoothCurvePoints[1]; // actual values of y
 
-      this.curveMinY = Math.min(...this.curvePoints);
-      this.curveMaxY = Math.max(...this.curvePoints);
-      this.gradient = this.GetGradient();
-    }
+    this.curveMinY = Math.min(...this.curvePoints);
+    this.curveMaxY = Math.max(...this.curvePoints);
+    this.gradient = this.GetGradient();
   }
 
   /**
@@ -5453,13 +5393,33 @@ class ScatterPlot {
   GetSvgPointCoords() {
     let points = new Map();
 
-    for (let i = 0; i < this.plotPoints.length; i++) {
-      let x = parseFloat(this.plotPoints[i].getAttribute(this.prefix + 'x')); // .toFixed(1);
-      let y = parseFloat(this.plotPoints[i].getAttribute(this.prefix + 'y'));
-      if (!points.has(x)) {
-        points.set(x, new Set([y]));
-      } else {
-        points.get(x).add(y);
+    if (this.plotPoints) {
+      for (let i = 0; i < this.plotPoints.length; i++) {
+        let x = parseFloat(this.plotPoints[i].getAttribute(this.prefix + 'x')); // .toFixed(1);
+        let y = parseFloat(this.plotPoints[i].getAttribute(this.prefix + 'y'));
+        if (!points.has(x)) {
+          points.set(x, new Set([y]));
+        } else {
+          points.get(x).add(y);
+        }
+      }
+    } else {
+      // pull from data instead
+      let elIndex = this.GetElementIndex('point');
+      for (let i = 0; i < singleMaidr.data[elIndex].length; i++) {
+        let x;
+        let y;
+        if ('x' in singleMaidr.data[elIndex][i]) {
+          x = singleMaidr.data[elIndex][i]['x'];
+        }
+        if ('y' in singleMaidr.data[elIndex][i]) {
+          y = singleMaidr.data[elIndex][i]['y'];
+        }
+        if (!points.has(x)) {
+          points.set(x, new Set([y]));
+        } else {
+          points.get(x).add(y);
+        }
       }
     }
 
@@ -5525,38 +5485,40 @@ class ScatterPlot {
 
     // but first, are we even in an svg that can be scaled?
     let isSvg = false;
-    let element = this.plotPoints[0]; // a random start, may as well be the first
-    while (element) {
-      if (element.tagName.toLowerCase() == 'body') {
-        break;
-      }
-      if (element.tagName && element.tagName.toLowerCase() === 'svg') {
-        isSvg = true;
-      }
-      element = element.parentNode;
-    }
-
-    if (isSvg) {
+    if (this.plotPoints) {
       let element = this.plotPoints[0]; // a random start, may as well be the first
       while (element) {
         if (element.tagName.toLowerCase() == 'body') {
           break;
         }
-        if (element.getAttribute('transform')) {
-          let transform = element.getAttribute('transform');
-          let match = transform.match(
-            /scale\((-?\d+(\.\d+)?),\s*(-?\d+(\.\d+)?)\)/
-          );
-          if (match) {
-            if (!isNaN(match[1])) {
-              scaleX *= parseFloat(match[1]);
-            }
-            if (!isNaN(match[3])) {
-              scaleY *= parseFloat(match[3]);
+        if (element.tagName && element.tagName.toLowerCase() === 'svg') {
+          isSvg = true;
+        }
+        element = element.parentNode;
+      }
+
+      if (isSvg) {
+        let element = this.plotPoints[0]; // a random start, may as well be the first
+        while (element) {
+          if (element.tagName.toLowerCase() == 'body') {
+            break;
+          }
+          if (element.getAttribute('transform')) {
+            let transform = element.getAttribute('transform');
+            let match = transform.match(
+              /scale\((-?\d+(\.\d+)?),\s*(-?\d+(\.\d+)?)\)/
+            );
+            if (match) {
+              if (!isNaN(match[1])) {
+                scaleX *= parseFloat(match[1]);
+              }
+              if (!isNaN(match[3])) {
+                scaleY *= parseFloat(match[3]);
+              }
             }
           }
+          element = element.parentNode;
         }
-        element = element.parentNode;
       }
     }
 
@@ -5565,22 +5527,30 @@ class ScatterPlot {
 
   /**
    * Returns a prefix based on the element type.
+   * This helps manipulate svg stuff, as the attribute info is slightly different depending on svg source
    * @returns {string} The prefix.
    */
   GetPrefix() {
     let pointIndex = this.GetElementIndex('point');
 
-    let element;
+    let element = null;
     if (pointIndex != -1) {
-      element = document.querySelectorAll(singleMaidr.selector[pointIndex])[0];
+      if ('selector' in singleMaidr) {
+        element = document.querySelectorAll(
+          singleMaidr.selector[pointIndex]
+        )[0];
+      } else if ('elements' in singleMaidr) {
+        element = singleMaidr.elements[pointIndex][0];
+      }
     } else if (singleMaidr.type == 'point') {
-      element = document.querySelectorAll(singleMaidr.selector)[0];
+      if ('selector' in singleMaidr) {
+        element = document.querySelectorAll(singleMaidr.selector)[0];
+      } else if ('elements' in singleMaidr) {
+        element = singleMaidr.elements[0];
+      }
     }
     let prefix = '';
-    if (
-      'selector' in singleMaidr &&
-      element.tagName.toLowerCase() === 'circle'
-    ) {
+    if (element && element.tagName.toLowerCase() === 'circle') {
       prefix = 'c';
     }
     return prefix;
@@ -5733,20 +5703,32 @@ class ScatterPlot {
    * @returns {Array<Array<number>>} An array containing two arrays: the x-coordinates and y-coordinates.
    */
   GetSvgLineCoords() {
-    // extract all the y coordinates from the point attribute of polyline
-    let str = this.plotLine.getAttribute('points');
-    let coords = str.split(' ');
-
-    let X = [];
-    let Y = [];
+    let x_points = [];
+    let y_points = [];
 
-    for (let i = 0; i < coords.length; i++) {
-      let coord = coords[i].split(',');
-      X.push(parseFloat(coord[0]));
-      Y.push(parseFloat(coord[1]));
+    if (this.plotLine) {
+      // extract all the y coordinates from the point attribute of polyline
+      let str = this.plotLine.getAttribute('points');
+      let coords = str.split(' ');
+      for (let i = 0; i < coords.length; i++) {
+        let coord = coords[i].split(',');
+        x_points.push(parseFloat(coord[0]));
+        y_points.push(parseFloat(coord[1]));
+      }
+    } else {
+      // fetch from data instead
+      let elIndex = this.GetElementIndex('point');
+      for (let i = 0; i < singleMaidr.data[elIndex].length; i++) {
+        if ('x' in singleMaidr.data[elIndex][i]) {
+          x_points.push(singleMaidr.data[elIndex][i]['x']);
+        }
+        if ('y' in singleMaidr.data[elIndex][i]) {
+          y_points.push(singleMaidr.data[elIndex][i]['y']);
+        }
+      }
     }
 
-    return [X, Y];
+    return [x, y];
   }
 
   /**
@@ -5808,6 +5790,24 @@ class ScatterPlot {
 
     return gradients;
   }
+
+  /**
+   * Returns whether or not we have elements / selectors for the given type.
+   * @param {string} type - The type of element to check for. eg, 'point' or 'smooth'.
+   * @returns {boolean} - True if we have elements / selectors for the given type, false otherwise.
+   * @function
+   * @memberof scatterplot
+   */
+  GetRectStatus(type) {
+    let elIndex = this.GetElementIndex(type);
+    if ('selector' in singleMaidr) {
+      return singleMaidr.selector[elIndex] ? true : false;
+    } else if ('elements' in singleMaidr) {
+      return singleMaidr.elements[elIndex] ? true : false;
+    } else {
+      return false;
+    }
+  }
 }
 
 /**
@@ -5826,6 +5826,7 @@ class Layer0Point {
     this.y = plot.chartPointsY[0];
     this.strokeWidth = 1.35;
     this.circleIndex = [];
+    this.hasRect = plot.GetRectStatus('point');
   }
 
   /**
@@ -5925,6 +5926,7 @@ class Layer1Point {
     this.x = plot.chartLineX[0];
     this.y = plot.chartLineY[0];
     this.strokeWidth = 1.35;
+    this.hasRect = plot.GetRectStatus('point');
   }
 
   /**
@@ -6015,6 +6017,8 @@ class Histogram {
     this.bars = null;
     if ('selector' in singleMaidr) {
       this.bars = document.querySelectorAll(singleMaidr.selector);
+    } else if ('elements' in singleMaidr) {
+      this.bars = singleMaidr.elements;
     }
 
     // labels (optional)
@@ -6164,57 +6168,7 @@ class LinePlot {
   constructor() {
     this.SetLineLayer();
     this.SetAxes();
-
-    let legendX = '';
-    let legendY = '';
-    if ('axes' in singleMaidr) {
-      // legend labels
-      if (singleMaidr.axes.x) {
-        if (singleMaidr.axes.x.label) {
-          if (legendX == '') {
-            legendX = singleMaidr.axes.x.label;
-          }
-        }
-      }
-      if (singleMaidr.axes.y) {
-        if (singleMaidr.axes.y.label) {
-          if (legendY == '') {
-            legendY = singleMaidr.axes.y.label;
-          }
-        }
-      }
-    }
-
-    this.plotLegend = {
-      x: legendX,
-      y: legendY,
-    };
-
-    // title
-    this.title = '';
-    if ('labels' in singleMaidr) {
-      if ('title' in singleMaidr.labels) {
-        this.title = singleMaidr.labels.title;
-      }
-    }
-    if (this.title == '') {
-      if ('title' in singleMaidr) {
-        this.title = singleMaidr.title;
-      }
-    }
-
-    // subtitle
-    if ('labels' in singleMaidr) {
-      if ('subtitle' in singleMaidr.labels) {
-        this.subtitle = singleMaidr.labels.subtitle;
-      }
-    }
-    // caption
-    if ('labels' in singleMaidr) {
-      if ('caption' in singleMaidr.labels) {
-        this.caption = singleMaidr.labels.caption;
-      }
-    }
+    this.UpdateConstants();
   }
 
   /**
@@ -6224,12 +6178,13 @@ class LinePlot {
     let elements;
     if ('selector' in singleMaidr) {
       elements = document.querySelectorAll(singleMaidr.selector);
+    } else if ('elements' in singleMaidr) {
+      elements = singleMaidr.elements;
     }
 
-    let len = elements.length;
-    this.plotLine = elements[len - 1];
+    if (elements) {
+      this.plotLine = elements[elements.length - 1];
 
-    if (typeof this.plotLine !== 'undefined') {
       let pointCoords = this.GetPointCoords();
       let pointValues = this.GetPoints();
 
@@ -6241,35 +6196,33 @@ class LinePlot {
 
       this.curveMinY = Math.min(...this.pointValuesY);
       this.curveMaxY = Math.max(...this.pointValuesY);
-      constants.minX = 0;
-      constants.maxX = this.pointValuesX.length - 1;
-      constants.minY = this.curveMinY;
-      constants.maxY = this.curveMaxY;
-
-      constants.autoPlayRate = Math.min(
-        Math.ceil(constants.AUTOPLAY_DURATION / (constants.maxX + 1)),
-        constants.MAX_SPEED
-      );
-      constants.DEFAULT_SPEED = constants.autoPlayRate;
-      if (constants.autoPlayRate < constants.MIN_SPEED) {
-        constants.MIN_SPEED = constants.autoPlayRate;
-      }
-
-      // this.gradient = this.GetGradient();
     }
   }
 
   /**
-   * Sets the minimum and maximum values for the x and y axes of a line plot.
+   * Updates the constants for the line plot.
+   * This includes the minimum and maximum x and y values, the autoplay rate, and the default speed.
    */
-  SetMinMax() {
+  UpdateConstants() {
     constants.minX = 0;
-    constants.maxX = this.pointValuesX.length - 1;
-    constants.minY = this.curveMinY;
-    constants.maxY = this.curveMaxY;
-    constants.autoPlayRate = Math.ceil(
-      constants.AUTOPLAY_DURATION / (constants.maxX + 1)
+    constants.maxX = singleMaidr.data.length - 1;
+    constants.minY = singleMaidr.data.reduce(
+      (min, item) => (item.y < min ? item.y : min),
+      singleMaidr.data[0].y
+    );
+    constants.maxY = singleMaidr.data.reduce(
+      (max, item) => (item.y > max ? item.y : max),
+      singleMaidr.data[0].y
     );
+
+    constants.autoPlayRate = Math.min(
+      Math.ceil(constants.AUTOPLAY_DURATION / (constants.maxX + 1)),
+      constants.MAX_SPEED
+    );
+    constants.DEFAULT_SPEED = constants.autoPlayRate;
+    if (constants.autoPlayRate < constants.MIN_SPEED) {
+      constants.MIN_SPEED = constants.autoPlayRate;
+    }
   }
 
   /**
@@ -6325,46 +6278,60 @@ class LinePlot {
     }
   }
 
-  // GetGradient() {
-  //   let gradients = [];
-
-  //   for (let i = 0; i < this.pointValuesY.length - 1; i++) {
-  //     let abs_grad = Math.abs(
-  //       (this.pointValuesY[i + 1] - this.pointValuesY[i]) /
-  //         (this.pointValuesX[i + 1] - this.pointValuesX[i])
-  //     ).toFixed(3);
-  //     gradients.push(abs_grad);
-  //   }
-
-  //   gradients.push('end');
-
-  //   return gradients;
-  // }
-
   /**
    * Sets the x and y group labels and title for the line plot based on the axes and title properties of the singleMaidr object.
    */
   SetAxes() {
-    this.x_group_label = '';
-    this.y_group_label = '';
-    this.title = '';
+    let legendX = '';
+    let legendY = '';
     if ('axes' in singleMaidr) {
-      if ('x' in singleMaidr.axes) {
-        if (this.x_group_label == '') {
-          this.x_group_label = singleMaidr.axes.x.label;
+      // legend labels
+      if (singleMaidr.axes.x) {
+        if (singleMaidr.axes.x.label) {
+          if (legendX == '') {
+            legendX = singleMaidr.axes.x.label;
+          }
         }
       }
-      if ('y' in singleMaidr.axes) {
-        if (this.y_group_label == '') {
-          this.y_group_label = singleMaidr.axes.y.label;
+      if (singleMaidr.axes.y) {
+        if (singleMaidr.axes.y.label) {
+          if (legendY == '') {
+            legendY = singleMaidr.axes.y.label;
+          }
         }
       }
     }
-    if ('title' in singleMaidr) {
-      if (this.title == '') {
+
+    this.plotLegend = {
+      x: legendX,
+      y: legendY,
+    };
+
+    // title
+    this.title = '';
+    if ('labels' in singleMaidr) {
+      if ('title' in singleMaidr.labels) {
+        this.title = singleMaidr.labels.title;
+      }
+    }
+    if (this.title == '') {
+      if ('title' in singleMaidr) {
         this.title = singleMaidr.title;
       }
     }
+
+    // subtitle
+    if ('labels' in singleMaidr) {
+      if ('subtitle' in singleMaidr.labels) {
+        this.subtitle = singleMaidr.labels.subtitle;
+      }
+    }
+    // caption
+    if ('labels' in singleMaidr) {
+      if ('caption' in singleMaidr.labels) {
+        this.caption = singleMaidr.labels.caption;
+      }
+    }
   }
 
   /**
@@ -6455,7 +6422,6 @@ class Segmented {
    */
   constructor() {
     // initialize variables level, data, and elements
-    let level = null;
     let fill = null;
     let data = null;
     let elements = null;
@@ -6463,17 +6429,17 @@ class Segmented {
       //axes.x.level
       if ('x' in singleMaidr.axes) {
         if ('level' in singleMaidr.axes.x) {
-          level = singleMaidr.axes.x.level;
+          this.level = singleMaidr.axes.x.level;
         }
       } else if ('y' in singleMaidr.axes) {
         if ('level' in singleMaidr.axes.y) {
-          level = singleMaidr.axes.y.level;
+          this.level = singleMaidr.axes.y.level;
         }
       }
       // axes.fill
       if ('fill' in singleMaidr.axes) {
         if ('level' in singleMaidr.axes.fill) {
-          fill = singleMaidr.axes.fill.level;
+          this.fill = singleMaidr.axes.fill.level;
         }
       }
     }
@@ -6482,6 +6448,8 @@ class Segmented {
     }
     if ('selector' in singleMaidr) {
       elements = document.querySelectorAll(singleMaidr.selector);
+    } else if ('elements' in singleMaidr) {
+      elements = singleMaidr.elements;
     }
 
     // gracefull failure: must have level + fill + data, elements optional
@@ -6489,9 +6457,10 @@ class Segmented {
       logError.LogAbsentElement('elements');
       constants.hasRect = 0;
     }
-    if (level != null && fill != null && data != null) {
-      this.level = level;
-      this.fill = fill.reverse(); // typically fill is in reverse order
+    if (data) {
+      if (this.fill) {
+        this.fill = this.fill.reverse(); // typically fill is in reverse order
+      }
       let dataAndELements = this.ParseData(data, elements);
       this.plotData = dataAndELements[0];
       this.elements = dataAndELements[1];
@@ -6579,7 +6548,12 @@ class Segmented {
     let plotData = [];
     let plotElements = [];
 
-    if (elements.length != data.length) {
+    // override and kill elements if not same length as data
+    if (elements) {
+      if (elements.length != data.length) {
+        plotElements = null;
+      }
+    } else {
       plotElements = null;
     }
 
@@ -6609,7 +6583,9 @@ class Segmented {
           // set actual values
           if (data[k].x == this.level[i] && data[k].fill == this.fill[j]) {
             plotData[i][j] = data[k].y;
-            plotElements[i][j] = elements[k];
+            if (elements) {
+              plotElements[i][j] = elements[k];
+            }
             break;
           }
         }
@@ -6660,6 +6636,7 @@ class Segmented {
       if (constants.sonifMode == 'on') {
         // we play a run of tones
         position.z = 0;
+        constants.KillSepPlay();
         constants.sepPlayId = setInterval(
           function () {
             // play this tone
@@ -7304,10 +7281,8 @@ class Control {
         window.position = new Position(-1, plot.plotData.length);
       }
       let rect;
-      constants.hasRect = false;
-      if ('selector' in singleMaidr) {
+      if (constants.hasRect) {
         rect = new BoxplotRect();
-        constants.hasRect = true;
       }
       let lastPlayed = '';
 
@@ -8256,7 +8231,7 @@ class Control {
         return new Promise((resolve) => setTimeout(resolve, time));
       }
 
-      // helper functions
+      // heat helper functions
       function lockPosition() {
         // lock to min / max postions
         let didLockHappen = false;
@@ -8659,7 +8634,7 @@ class Control {
         if (constants.showDisplay) {
           display.displayValues();
         }
-        if (constants.showRect) {
+        if (layer0Point.hasRect) {
           layer0Point.UpdatePointDisplay();
         }
         if (constants.sonifMode != 'off') {
@@ -8672,9 +8647,9 @@ class Control {
           display.displayValues();
         }
         if (constants.showRect) {
-          if (constants.chartType == 'point') {
+          if (constants.chartType == 'point' && layer0Point.hasRect) {
             layer0Point.UpdatePointDisplay();
-          } else {
+          } else if (constants.chartType == 'smooth' && layer1Point.hasRect) {
             layer1Point.UpdatePointDisplay();
           }
         }
@@ -8689,7 +8664,7 @@ class Control {
         if (constants.showDisplayInBraille) {
           display.displayValues();
         }
-        if (constants.showRect) {
+        if (layer1Point.hasRect) {
           layer1Point.UpdatePointDisplay();
         }
         if (constants.sonifMode != 'off') {