maidr 2.4.0 → 2.5.1

package/README.md

@@ -13,11 +13,15 @@ maidr (pronounced as 'mader') is a system for non-visual access and control of s
 ## Table of Contents
 
 1. [Usage](#usage)
-2. [Controls](#controls)
-3. [Braille Generation](#braille-generation)
-4. [License](#license)
-5. [Contact](#contact)
-6. [Acknowledgments](#acknowledgments)
+1. [Controls](#controls)
+1. [Braille Generation](#braille-generation)
+1. [API](#api)
+1. [Binders](#binders)
+1. [Papers](#papers)
+1. [License](#license)
+1. [Contact](#contact)
+1. [Acknowledgments](#acknowledgments)
+
 
 ## Usage
 
@@ -47,7 +51,7 @@ To use maidr, follow these steps:
    </html>
    ```
 
-3. Add your data: Include your data as a json schema directly in the HTML file. There should be a single `maidr` object with the following properties, or an array of objects if multiple charts exist on the page. Your json schema may look like so: (values for demonstration purposes)
+3. Add your data: Include your data as a json schema directly in the HTML file. There should be a single `maidr` object with the following properties, or an array of objects if multiple plots exist on the page. Your json schema may look like so: (values for demonstration purposes)
 
    ```javascript
    // a single plot
@@ -74,7 +78,7 @@ To use maidr, follow these steps:
      data: ...
    }
 
-   // or, multiple charts
+   // or, multiple plots
    let maidr = [
      {
        type: 'box',
@@ -220,7 +224,7 @@ For more information and examples, refer to the example HTML files provided in t
 
 ## Controls
 
-To interact with the charts using maidr, follow these steps:
+To interact with the plots using maidr, follow these steps:
 
 1. Press the **Tab** key to focus on the SVG element.
 2. Use the **arrow keys** to move around the plot.
@@ -355,6 +359,53 @@ In the braille representation of segmented bar plots, braille depends on where y
 
 In the Braille representation of a lineplot, braille is nearly identical to the above barplot: data values are encoded as Braille characters based on their relative magnitude within the plot. Low values are denoted by Braille characters that have dots only along the bottom, while high values are indicated by characters that have dots higher up.
 
+## API
+
+maidr is available via a restful API. Learn more about the usage at [maidr-api](https://github.com/xability/maidr-api) repo.
+
+## Binders
+
+We currently provide the following binders, all of which can be found at each separate repo:
+
+- [x] Python binder for matplotlib and seaborn: [py_maidr](https://github.com/xability/py_maidr).
+
+- [ ] R binder for ggplot2: [r_maidr](https://github.com/xability/r_maidr).
+
+## Papers
+
+To learn more about the theoretical background and user study results, we recommend you read and cite the following papers.
+
+1. [MAIDR: Making Statistical Visualizations Accessible with Multimodal Data Representation](https://arxiv.org/abs/2403.00717):
+
+```tex
+@inproceedings{seoMAIDR2024,
+  title     = {{{MAIDR}}: Making Statistical Visualizations Accessible with Multimodal Data Representation},
+  booktitle = {Proceedings of the {{SIGCHI Conference}} on {{Human Factors}} in {{Computing Systems}}},
+  author    = {Seo, JooYoung and Xia, Yilin and Lee, Bongshin and McCurry, Sean and Yam, Yu Jun},
+  year      = {2024},
+  doi       = {10.1145/3613904.3642730}
+}
+```
+
+1. [Born Accessible Data Science and Visualization Courses: Challenges of Developing Curriculum to be Taught by Blind Instructors to Blind Students](https://arxiv.org/abs/2403.02568v1):
+
+```tex
+@misc{seoBornAccessibleData2024,
+  title         = {Born {{Accessible Data Science}} and {{Visualization Courses}}: {{Challenges}} of {{Developing Curriculum}} to Be {{Taught}} by {{Blind Instructors}} to {{Blind Students}}},
+  shorttitle    = {Born {{Accessible Data Science}} and {{Visualization Courses}}},
+  author        = {Seo, JooYoung and O'Modhrain, Sile and Xia, Yilin and Kamath, Sanchita and Lee, Bongshin and Coughlan, James M.},
+  year          = {2024},
+  month         = mar,
+  number        = {arXiv:2403.02568},
+  eprint        = {2403.02568},
+  primaryclass  = {cs},
+  publisher     = {{arXiv}},
+  urldate       = {2024-03-08},
+  archiveprefix = {arxiv},
+  keywords      = {Computer Science - Human-Computer Interaction}
+}
+```
+
 ## License
 
 This project is licensed under the GPL 3 License.

package/dist/maidr.js

@@ -86,6 +86,7 @@ class Constants {
     'You are a helpful assistant describing the chart to a blind person. ';
   skillLevel = 'basic'; // basic / intermediate / expert
   skillLevelOther = ''; // custom skill level
+  autoInitLLM = true; // auto initialize LLM on page load
 
   // user controls (not exposed to menu, with shortcuts usually)
   showDisplay = 1; // true / false
@@ -429,6 +430,9 @@ class Menu {
                               <span id="gemini_multi_container" class="hidden"><input type="checkbox" id="gemini_multi" name="gemini_multi" aria-label="Use Gemini in Multi modal mode"></span>
                               <input type="password" id="gemini_auth_key"><button aria-label="Delete Gemini key" title="Delete Gemini key" id="delete_gemini_key" class="invis_button">&times;</button><label for="gemini_auth_key">Gemini Authentication Key</label>
                             </p>
+                            <p><input type="checkbox" ${
+                              constants.autoInitLLM ? 'checked' : ''
+                            } id="init_llm_on_load" name="init_llm_on_load"><label for="init_llm_on_load">Start first LLM chat chart load</label></p>
                             <p>
                                 <select id="skill_level">
                                     <option value="basic">Basic</option>
@@ -740,7 +744,7 @@ class Menu {
    * Saves the data from the HTML elements into the constants object.
    */
   SaveData() {
-    this.HandleLLMChanges();
+    let shouldReset = this.ShouldLLMReset();
 
     constants.vol = document.getElementById('vol').value;
     constants.autoPlayRate = document.getElementById('autoplay_rate').value;
@@ -760,9 +764,9 @@ class Menu {
       document.getElementById('skill_level_other').value;
     constants.LLMModel = document.getElementById('LLM_model').value;
     constants.LLMPreferences = document.getElementById('LLM_preferences').value;
-
     constants.LLMOpenAiMulti = document.getElementById('openai_multi').checked;
     constants.LLMGeminiMulti = document.getElementById('gemini_multi').checked;
+    constants.autoInitLLM = document.getElementById('init_llm_on_load').checked;
 
     // aria
     if (document.getElementById('aria_mode_assertive').checked) {
@@ -773,6 +777,12 @@ class Menu {
 
     this.SaveDataToLocalStorage();
     this.UpdateHtml();
+
+    if (shouldReset) {
+      if (chatLLM) {
+        chatLLM.ResetLLM();
+      }
+    }
   }
 
   /**
@@ -789,6 +799,8 @@ class Menu {
     document
       .getElementById(constants.announcement_container_id)
       .setAttribute('aria-live', constants.ariaMode);
+
+    document.getElementById('init_llm_on_load').checked = constants.autoInitLLM;
   }
 
   /**
@@ -797,6 +809,10 @@ class Menu {
   NotifyOfLLMReset() {
     let html =
       '<p id="LLM_reset_notification">Note: Changes in LLM settings will reset any existing conversation.</p>';
+
+    if (document.getElementById('LLM_reset_notification')) {
+      document.getElementById('LLM_reset_notification').remove();
+    }
     document
       .getElementById('save_and_close_menu')
       .insertAdjacentHTML('beforebegin', html);
@@ -813,7 +829,7 @@ class Menu {
    * Handles changes to the LLM model and multi-modal settings.
    * We reset if we change the LLM model, multi settings, or skill level.
    */
-  HandleLLMChanges() {
+  ShouldLLMReset() {
     let shouldReset = false;
     if (
       !shouldReset &&
@@ -837,11 +853,7 @@ class Menu {
       shouldReset = true;
     }
 
-    if (shouldReset) {
-      if (chatLLM) {
-        chatLLM.ResetChatHistory();
-      }
-    }
+    return shouldReset;
   }
 
   /**
@@ -868,6 +880,7 @@ class Menu {
     data.LLMPreferences = constants.LLMPreferences;
     data.LLMOpenAiMulti = constants.LLMOpenAiMulti;
     data.LLMGeminiMulti = constants.LLMGeminiMulti;
+    data.autoInitLLM = constants.autoInitLLM;
     localStorage.setItem('settings_data', JSON.stringify(data));
   }
   /**
@@ -892,6 +905,7 @@ class Menu {
       constants.LLMPreferences = data.LLMPreferences;
       constants.LLMOpenAiMulti = data.LLMOpenAiMulti;
       constants.LLMGeminiMulti = data.LLMGeminiMulti;
+      constants.autoInitLLM = data.autoInitLLM;
     }
     this.PopulateData();
     this.UpdateHtml();
@@ -907,8 +921,12 @@ class ChatLLM {
   constructor() {
     this.firstTime = true;
     this.firstMulti = true;
+    this.shown = false;
     this.CreateComponent();
     this.SetEvents();
+    if (constants.autoInitLLM) {
+      this.InitChatMessage();
+    }
   }
 
   /**
@@ -1073,10 +1091,18 @@ class ChatLLM {
       document.getElementById('reset_chatLLM'),
       'click',
       function (e) {
-        chatLLM.ResetChatHistory();
+        chatLLM.ResetLLM();
       },
     ]);
 
+    // bookmark:
+    //
+    // have LLM run on init (#425)
+    // quiet first, but if they open window, it does the beep and aria live alert
+    // make toggle in settings to yes / no auto initiate LLM (or wait for window to open)
+    // as part of this, fix reset so it loads the LLM again without refreshing the window,
+    // left undone from the other request
+
     // copy to clipboard
     constants.events.push([
       document.getElementById('chatLLM_copy_all'),
@@ -1123,7 +1149,7 @@ class ChatLLM {
   async Submit(text, firsttime = false) {
     // start waiting sound
     if (constants.playLLMWaitingSound) {
-      chatLLM.WaitingSound(true);
+      this.WaitingSound(true);
     }
 
     let img = null;
@@ -1169,7 +1195,7 @@ class ChatLLM {
       let delay = 1000;
       let freq = 440; // a440 babee
       constants.waitingInterval = setInterval(function () {
-        if (audio) {
+        if (audio && chatLLM.shown) {
           audio.playOscillator(freq, 0.2, 0);
         }
       }, delay);
@@ -1181,6 +1207,15 @@ class ChatLLM {
     }
   }
 
+  InitChatMessage() {
+    // get name from resource
+    let LLMName = resources.GetString(constants.LLMModel);
+    this.firstTime = false;
+    this.DisplayChatMessage(LLMName, resources.GetString('processing'), true);
+    let defaultPrompt = this.GetDefaultPrompt();
+    this.Submit(defaultPrompt, true);
+  }
+
   /**
    * Processes the response from the LLM and displays it to the user.
    * @function
@@ -1392,9 +1427,9 @@ class ChatLLM {
       };
 
       // Generate the content
-      //console.log('LLM request: ', prompt, image);
+      console.log('LLM request: ', prompt, image);
       const result = await model.generateContent([prompt, image]);
-      //console.log(result.response.text());
+      console.log(result.response.text());
 
       // Process the response
       chatLLM.ProcessLLMResponse(result.response, 'gemini');
@@ -1457,7 +1492,7 @@ class ChatLLM {
   /**
    * Resets the chat history window
    */
-  ResetChatHistory() {
+  ResetLLM() {
     // clear the main chat history
     document.getElementById('chatLLM_chat_history').innerHTML = '';
     // unhide the more button
@@ -1469,6 +1504,11 @@ class ChatLLM {
     // reset the data
     this.requestJson = null;
     this.firstTime = true;
+
+    // and start over, if enabled
+    if (constants.autoInitLLM) {
+      chatLLM.InitChatMessage();
+    }
   }
 
   /**
@@ -1502,6 +1542,7 @@ class ChatLLM {
         onoff = false;
       }
     }
+    chatLLM.shown = onoff;
     if (onoff) {
       // open
       this.whereWasMyFocus = document.activeElement;
@@ -1511,20 +1552,6 @@ class ChatLLM {
         .getElementById('chatLLM_modal_backdrop')
         .classList.remove('hidden');
       document.querySelector('#chatLLM .close').focus();
-
-      // first time, send default query
-      if (this.firstTime) {
-        // get name from resource
-        let LLMName = resources.GetString(constants.LLMModel);
-        this.firstTime = false;
-        this.DisplayChatMessage(
-          LLMName,
-          resources.GetString('processing'),
-          true
-        );
-        let defaultPrompt = this.GetDefaultPrompt();
-        this.Submit(defaultPrompt, true);
-      }
     } else {
       // close
       document.getElementById('chatLLM').classList.add('hidden');
@@ -6631,19 +6658,21 @@ class LinePlot {
 
     if (elements) {
       this.plotLine = elements[elements.length - 1];
+    } else {
+      constants.hasRect = 0;
+    }
 
-      let pointCoords = this.GetPointCoords();
-      let pointValues = this.GetPoints();
+    let pointCoords = this.GetPointCoords();
+    let pointValues = this.GetPoints();
 
-      this.chartLineX = pointCoords[0]; // x coordinates of curve
-      this.chartLineY = pointCoords[1]; // y coordinates of curve
+    this.chartLineX = pointCoords[0]; // x coordinates of curve
+    this.chartLineY = pointCoords[1]; // y coordinates of curve
 
-      this.pointValuesX = pointValues[0]; // actual values of x
-      this.pointValuesY = pointValues[1]; // actual values of y
+    this.pointValuesX = pointValues[0]; // actual values of x
+    this.pointValuesY = pointValues[1]; // actual values of y
 
-      this.curveMinY = Math.min(...this.pointValuesY);
-      this.curveMaxY = Math.max(...this.pointValuesY);
-    }
+    this.curveMinY = Math.min(...this.pointValuesY);
+    this.curveMaxY = Math.max(...this.pointValuesY);
   }
 
   /**
@@ -6678,27 +6707,47 @@ class LinePlot {
    */
   GetPointCoords() {
     let svgLineCoords = [[], []];
-    // lineplot SVG containing path element instead of polyline
-    if (this.plotLine instanceof SVGPathElement) {
-      // Assuming the path data is in the format "M x y L x y L x y L x y"
-      const pathD = this.plotLine.getAttribute('d');
-      const regex = /[ML]\s*(-?\d+(\.\d+)?)\s+(-?\d+(\.\d+)?)/g;
 
-      let match;
-      while ((match = regex.exec(pathD)) !== null) {
-        svgLineCoords[0].push(match[1]); // x coordinate
-        svgLineCoords[1].push(match[3]); // y coordinate
+    if (this.plotLine) {
+      // lineplot SVG containing path element instead of polyline
+      if (this.plotLine instanceof SVGPathElement) {
+        // Assuming the path data is in the format "M x y L x y L x y L x y"
+        const pathD = this.plotLine.getAttribute('d');
+        const regex = /[ML]\s*(-?\d+(\.\d+)?)\s+(-?\d+(\.\d+)?)/g;
+
+        let match;
+        while ((match = regex.exec(pathD)) !== null) {
+          svgLineCoords[0].push(match[1]); // x coordinate
+          svgLineCoords[1].push(match[3]); // y coordinate
+        }
+      } else {
+        let points = this.plotLine.getAttribute('points').split(' ');
+        for (let i = 0; i < points.length; i++) {
+          if (points[i] !== '') {
+            let point = points[i].split(',');
+            svgLineCoords[0].push(point[0]);
+            svgLineCoords[1].push(point[1]);
+          }
+        }
       }
     } else {
-      let points = this.plotLine.getAttribute('points').split(' ');
-      for (let i = 0; i < points.length; i++) {
-        if (points[i] !== '') {
-          let point = points[i].split(',');
-          svgLineCoords[0].push(point[0]);
-          svgLineCoords[1].push(point[1]);
+      // fetch from data instead
+      let x_points = [];
+      let y_points = [];
+
+      let data;
+      if ('data' in singleMaidr) {
+        data = singleMaidr.data;
+      }
+      if (typeof data !== 'undefined') {
+        for (let i = 0; i < data.length; i++) {
+          x_points.push(data[i].x);
+          y_points.push(data[i].y);
         }
       }
+      return [x_points, y_points];
     }
+
     return svgLineCoords;
   }
 
@@ -6720,8 +6769,6 @@ class LinePlot {
         y_points.push(data[i].y);
       }
       return [x_points, y_points];
-    } else {
-      return;
     }
   }
 
@@ -10220,7 +10267,7 @@ class Control {
         if (constants.showDisplay) {
           display.displayValues();
         }
-        if (constants.showRect) {
+        if (constants.showRect && constants.hasRect) {
           point.UpdatePointDisplay();
         }
         if (constants.sonifMode != 'off') {