maidr 2.3.1 → 2.5.0

package/README.md

@@ -13,11 +13,15 @@ maidr (pronounced as 'mader') is a system for non-visual access and control of s
 ## Table of Contents
 
 1. [Usage](#usage)
-2. [Controls](#controls)
-3. [Braille Generation](#braille-generation)
-4. [License](#license)
-5. [Contact](#contact)
-6. [Acknowledgments](#acknowledgments)
+1. [Controls](#controls)
+1. [Braille Generation](#braille-generation)
+1. [API](#api)
+1. [Binders](#binders)
+1. [Papers](#papers)
+1. [License](#license)
+1. [Contact](#contact)
+1. [Acknowledgments](#acknowledgments)
+
 
 ## Usage
 
@@ -47,7 +51,7 @@ To use maidr, follow these steps:
    </html>
    ```
 
-3. Add your data: Include your data as a json schema directly in the HTML file. There should be a single `maidr` object with the following properties, or an array of objects if multiple charts exist on the page. Your json schema may look like so: (values for demonstration purposes)
+3. Add your data: Include your data as a json schema directly in the HTML file. There should be a single `maidr` object with the following properties, or an array of objects if multiple plots exist on the page. Your json schema may look like so: (values for demonstration purposes)
 
    ```javascript
    // a single plot
@@ -74,7 +78,7 @@ To use maidr, follow these steps:
      data: ...
    }
 
-   // or, multiple charts
+   // or, multiple plots
    let maidr = [
      {
        type: 'box',
@@ -220,7 +224,7 @@ For more information and examples, refer to the example HTML files provided in t
 
 ## Controls
 
-To interact with the charts using maidr, follow these steps:
+To interact with the plots using maidr, follow these steps:
 
 1. Press the **Tab** key to focus on the SVG element.
 2. Use the **arrow keys** to move around the plot.
@@ -355,6 +359,53 @@ In the braille representation of segmented bar plots, braille depends on where y
 
 In the Braille representation of a lineplot, braille is nearly identical to the above barplot: data values are encoded as Braille characters based on their relative magnitude within the plot. Low values are denoted by Braille characters that have dots only along the bottom, while high values are indicated by characters that have dots higher up.
 
+## API
+
+maidr is available via a restful API. Learn more about the usage at [maidr-api](https://github.com/xability/maidr-api) repo.
+
+## Binders
+
+We currently provide the following binders, all of which can be found at each separate repo:
+
+- [x] Python binder for matplotlib and seaborn: [py_maidr](https://github.com/xability/py_maidr).
+
+- [ ] R binder for ggplot2: [r_maidr](https://github.com/xability/r_maidr).
+
+## Papers
+
+To learn more about the theoretical background and user study results, we recommend you read and cite the following papers.
+
+1. [MAIDR: Making Statistical Visualizations Accessible with Multimodal Data Representation](https://arxiv.org/abs/2403.00717):
+
+```tex
+@inproceedings{seoMAIDR2024,
+  title     = {{{MAIDR}}: Making Statistical Visualizations Accessible with Multimodal Data Representation},
+  booktitle = {Proceedings of the {{SIGCHI Conference}} on {{Human Factors}} in {{Computing Systems}}},
+  author    = {Seo, JooYoung and Xia, Yilin and Lee, Bongshin and McCurry, Sean and Yam, Yu Jun},
+  year      = {2024},
+  doi       = {10.1145/3613904.3642730}
+}
+```
+
+1. [Born Accessible Data Science and Visualization Courses: Challenges of Developing Curriculum to be Taught by Blind Instructors to Blind Students](https://arxiv.org/abs/2403.02568v1):
+
+```tex
+@misc{seoBornAccessibleData2024,
+  title         = {Born {{Accessible Data Science}} and {{Visualization Courses}}: {{Challenges}} of {{Developing Curriculum}} to Be {{Taught}} by {{Blind Instructors}} to {{Blind Students}}},
+  shorttitle    = {Born {{Accessible Data Science}} and {{Visualization Courses}}},
+  author        = {Seo, JooYoung and O'Modhrain, Sile and Xia, Yilin and Kamath, Sanchita and Lee, Bongshin and Coughlan, James M.},
+  year          = {2024},
+  month         = mar,
+  number        = {arXiv:2403.02568},
+  eprint        = {2403.02568},
+  primaryclass  = {cs},
+  publisher     = {{arXiv}},
+  urldate       = {2024-03-08},
+  archiveprefix = {arxiv},
+  keywords      = {Computer Science - Human-Computer Interaction}
+}
+```
+
 ## License
 
 This project is licensed under the GPL 3 License.

package/dist/maidr.js

@@ -86,6 +86,7 @@ class Constants {
     'You are a helpful assistant describing the chart to a blind person. ';
   skillLevel = 'basic'; // basic / intermediate / expert
   skillLevelOther = ''; // custom skill level
+  autoInitLLM = true; // auto initialize LLM on page load
 
   // user controls (not exposed to menu, with shortcuts usually)
   showDisplay = 1; // true / false
@@ -273,6 +274,7 @@ class Resources {
         openai: 'OpenAI Vision',
         gemini: 'Gemini Pro Vision',
         multi: 'Multiple AI',
+        processing: 'Processing Chart...',
       },
     },
   };
@@ -428,6 +430,9 @@ class Menu {
                               <span id="gemini_multi_container" class="hidden"><input type="checkbox" id="gemini_multi" name="gemini_multi" aria-label="Use Gemini in Multi modal mode"></span>
                               <input type="password" id="gemini_auth_key"><button aria-label="Delete Gemini key" title="Delete Gemini key" id="delete_gemini_key" class="invis_button">&times;</button><label for="gemini_auth_key">Gemini Authentication Key</label>
                             </p>
+                            <p><input type="checkbox" ${
+                              constants.autoInitLLM ? 'checked' : ''
+                            } id="init_llm_on_load" name="init_llm_on_load"><label for="init_llm_on_load">Start first LLM chat chart load</label></p>
                             <p>
                                 <select id="skill_level">
                                     <option value="basic">Basic</option>
@@ -443,7 +448,7 @@ class Menu {
                         </div>
                     </div>
                     <div class="modal-footer">
-                        <button type="button" id="save_and_close_menu">Save and Close</button>
+                        <button type="button" id="save_and_close_menu" aria-labelledby="save_and_close_text"><span id="save_and_close_text">Save and Close</span></button>
                         <button type="button" id="close_menu">Close</button>
                     </div>
                 </div>
@@ -579,6 +584,16 @@ class Menu {
         }
       },
     ]);
+
+    // trigger notification that LLM will be reset
+    // this is done on change of LLM model, multi settings, or skill level
+    constants.events.push([
+      document.getElementById('LLM_model'),
+      'change',
+      function (e) {
+        menu.NotifyOfLLMReset();
+      },
+    ]);
   }
 
   /**
@@ -720,13 +735,16 @@ class Menu {
       document.getElementById('LLM_preferences').value =
         constants.LLMPreferences;
     }
+    if (document.getElementById('LLM_reset_notification')) {
+      document.getElementById('LLM_reset_notification').remove();
+    }
   }
 
   /**
    * Saves the data from the HTML elements into the constants object.
    */
   SaveData() {
-    this.HandleLLMChanges();
+    let shouldReset = this.ShouldLLMReset();
 
     constants.vol = document.getElementById('vol').value;
     constants.autoPlayRate = document.getElementById('autoplay_rate').value;
@@ -746,9 +764,9 @@ class Menu {
       document.getElementById('skill_level_other').value;
     constants.LLMModel = document.getElementById('LLM_model').value;
     constants.LLMPreferences = document.getElementById('LLM_preferences').value;
-
     constants.LLMOpenAiMulti = document.getElementById('openai_multi').checked;
     constants.LLMGeminiMulti = document.getElementById('gemini_multi').checked;
+    constants.autoInitLLM = document.getElementById('init_llm_on_load').checked;
 
     // aria
     if (document.getElementById('aria_mode_assertive').checked) {
@@ -759,6 +777,12 @@ class Menu {
 
     this.SaveDataToLocalStorage();
     this.UpdateHtml();
+
+    if (shouldReset) {
+      if (chatLLM) {
+        chatLLM.ResetLLM();
+      }
+    }
   }
 
   /**
@@ -775,13 +799,37 @@ class Menu {
     document
       .getElementById(constants.announcement_container_id)
       .setAttribute('aria-live', constants.ariaMode);
+
+    document.getElementById('init_llm_on_load').checked = constants.autoInitLLM;
   }
 
+  /**
+   * Notifies the user that the LLM will be reset.
+   */
+  NotifyOfLLMReset() {
+    let html =
+      '<p id="LLM_reset_notification">Note: Changes in LLM settings will reset any existing conversation.</p>';
+
+    if (document.getElementById('LLM_reset_notification')) {
+      document.getElementById('LLM_reset_notification').remove();
+    }
+    document
+      .getElementById('save_and_close_menu')
+      .insertAdjacentHTML('beforebegin', html);
+
+    // add to aria button text
+    document
+      .getElementById('save_and_close_menu')
+      .setAttribute(
+        'aria-labelledby',
+        'save_and_close_text LLM_reset_notification'
+      );
+  }
   /**
    * Handles changes to the LLM model and multi-modal settings.
    * We reset if we change the LLM model, multi settings, or skill level.
    */
-  HandleLLMChanges() {
+  ShouldLLMReset() {
     let shouldReset = false;
     if (
       !shouldReset &&
@@ -805,11 +853,7 @@ class Menu {
       shouldReset = true;
     }
 
-    if (shouldReset) {
-      if (chatLLM) {
-        chatLLM.ResetChatHistory();
-      }
-    }
+    return shouldReset;
   }
 
   /**
@@ -836,6 +880,7 @@ class Menu {
     data.LLMPreferences = constants.LLMPreferences;
     data.LLMOpenAiMulti = constants.LLMOpenAiMulti;
     data.LLMGeminiMulti = constants.LLMGeminiMulti;
+    data.autoInitLLM = constants.autoInitLLM;
     localStorage.setItem('settings_data', JSON.stringify(data));
   }
   /**
@@ -860,6 +905,7 @@ class Menu {
       constants.LLMPreferences = data.LLMPreferences;
       constants.LLMOpenAiMulti = data.LLMOpenAiMulti;
       constants.LLMGeminiMulti = data.LLMGeminiMulti;
+      constants.autoInitLLM = data.autoInitLLM;
     }
     this.PopulateData();
     this.UpdateHtml();
@@ -875,8 +921,12 @@ class ChatLLM {
   constructor() {
     this.firstTime = true;
     this.firstMulti = true;
+    this.shown = false;
     this.CreateComponent();
     this.SetEvents();
+    if (constants.autoInitLLM) {
+      this.InitChatMessage();
+    }
   }
 
   /**
@@ -895,8 +945,11 @@ class ChatLLM {
                         </button>
                     </div>
                     <div class="modal-body">
+                        <div id="chatLLM_chat_history_wrapper">
                         <div id="chatLLM_chat_history" aria-live="${constants.ariaMode}" aria-relevant="additions">
                         </div>
+                        <p id="chatLLM_copy_all_wrapper"><button id="chatLLM_copy_all">Copy all to clipboard</button></p>
+                        </div>
                         <div id="chatLLM_content">
                           <p><input type="text" id="chatLLM_input" class="form-control" name="chatLLM_input" aria-labelledby="chatLLM_title" size="50"></p>
                           <div class="LLM_suggestions">
@@ -1038,8 +1091,47 @@ class ChatLLM {
       document.getElementById('reset_chatLLM'),
       'click',
       function (e) {
-        chatLLM.Toggle(false);
-        chatLLM.ResetChatHistory();
+        chatLLM.ResetLLM();
+      },
+    ]);
+
+    // bookmark:
+    //
+    // have LLM run on init (#425)
+    // quiet first, but if they open window, it does the beep and aria live alert
+    // make toggle in settings to yes / no auto initiate LLM (or wait for window to open)
+    // as part of this, fix reset so it loads the LLM again without refreshing the window,
+    // left undone from the other request
+
+    // copy to clipboard
+    constants.events.push([
+      document.getElementById('chatLLM_copy_all'),
+      'click',
+      function (e) {
+        let text = document.getElementById('chatLLM_chat_history').innerText;
+        // need newlines instead of paragraphs headings etc
+        text = text.replace(/<p>/g, '\n').replace(/<\/p>/g, '\n');
+        text = text.replace(/<h\d>/g, '\n').replace(/<\/h\d>/g, '\n');
+        text = text.replace(/<.*?>/g, '');
+
+        navigator.clipboard.writeText(text);
+      },
+    ]);
+    constants.events.push([
+      document.getElementById('chatLLM_chat_history'),
+      'click',
+      function (e) {
+        // we're delegating here, so set the event on child .chatLLM_message_copy_button
+        if (e.target.matches('.chatLLM_message_copy_button')) {
+          // get the innerText of the element before the button
+          let text = e.target.closest('p').previousElementSibling.innerText;
+          // need newlines instead of paragraphs headings etc
+          text = text.replace(/<p>/g, '\n').replace(/<\/p>/g, '\n');
+          text = text.replace(/<h\d>/g, '\n').replace(/<\/h\d>/g, '\n');
+          text = text.replace(/<.*?>/g, '');
+
+          navigator.clipboard.writeText(text);
+        }
       },
     ]);
   }
@@ -1057,7 +1149,7 @@ class ChatLLM {
   async Submit(text, firsttime = false) {
     // start waiting sound
     if (constants.playLLMWaitingSound) {
-      chatLLM.WaitingSound(true);
+      this.WaitingSound(true);
     }
 
     let img = null;
@@ -1103,7 +1195,7 @@ class ChatLLM {
       let delay = 1000;
       let freq = 440; // a440 babee
       constants.waitingInterval = setInterval(function () {
-        if (audio) {
+        if (audio && chatLLM.shown) {
           audio.playOscillator(freq, 0.2, 0);
         }
       }, delay);
@@ -1115,6 +1207,15 @@ class ChatLLM {
     }
   }
 
+  InitChatMessage() {
+    // get name from resource
+    let LLMName = resources.GetString(constants.LLMModel);
+    this.firstTime = false;
+    this.DisplayChatMessage(LLMName, resources.GetString('processing'), true);
+    let defaultPrompt = this.GetDefaultPrompt();
+    this.Submit(defaultPrompt, true);
+  }
+
   /**
    * Processes the response from the LLM and displays it to the user.
    * @function
@@ -1122,7 +1223,7 @@ class ChatLLM {
    */
   ProcessLLMResponse(data, model) {
     chatLLM.WaitingSound(false);
-    console.log('LLM response: ', data);
+    //console.log('LLM response: ', data);
     let text = '';
     let LLMName = resources.GetString(model);
 
@@ -1368,6 +1469,12 @@ class ChatLLM {
         <p class="chatLLM_message_text">${text}</p>
       </div>
     `;
+    // add a copy button to actual messages
+    if (user != 'User' && text != resources.GetString('processing')) {
+      html += `
+        <p class="chatLLM_message_copy"><button class="chatLLM_message_copy_button">Copy</button></p>
+      `;
+    }
 
     this.RenderChatMessage(html);
   }
@@ -1385,7 +1492,7 @@ class ChatLLM {
   /**
    * Resets the chat history window
    */
-  ResetChatHistory() {
+  ResetLLM() {
     // clear the main chat history
     document.getElementById('chatLLM_chat_history').innerHTML = '';
     // unhide the more button
@@ -1397,6 +1504,11 @@ class ChatLLM {
     // reset the data
     this.requestJson = null;
     this.firstTime = true;
+
+    // and start over, if enabled
+    if (constants.autoInitLLM) {
+      chatLLM.InitChatMessage();
+    }
   }
 
   /**
@@ -1430,6 +1542,7 @@ class ChatLLM {
         onoff = false;
       }
     }
+    chatLLM.shown = onoff;
     if (onoff) {
       // open
       this.whereWasMyFocus = document.activeElement;
@@ -1439,16 +1552,6 @@ class ChatLLM {
         .getElementById('chatLLM_modal_backdrop')
         .classList.remove('hidden');
       document.querySelector('#chatLLM .close').focus();
-
-      // first time, send default query
-      if (this.firstTime) {
-        // get name from resource
-        let LLMName = resources.GetString(constants.LLMModel);
-        this.firstTime = false;
-        this.DisplayChatMessage(LLMName, 'Processing Chart...', true);
-        let defaultPrompt = this.GetDefaultPrompt();
-        this.Submit(defaultPrompt, true);
-      }
     } else {
       // close
       document.getElementById('chatLLM').classList.add('hidden');