maidr 2.9.1 → 2.10.0

package/README.md

@@ -8,7 +8,7 @@
 
 # maidr: Multimodal Access and Interactive Data Representation
 
-maidr (pronounced as 'mader') is a system for non-visual access and control of statistical plots. It aims to provide an inclusive experience for users with visual impairments by offering multiple modes of interaction: braille, text, and sonification (BTS). This comprehensive approach enhances the accessibility of data visualization and encourages a multi-modal exploration on visualization. Check out the current build: [maidr Demo](https://xability.github.io/maidr/user_study_pilot/intro.html). You may also clone or download the GitHub repo, navigate to the ./user_study_pilot folder, and open any of the html files in your browser.
+maidr (pronounced as 'mader') is a system for non-visual access and control of statistical plots. It aims to provide an inclusive experience for users with visual impairments by offering multiple modes of interaction: braille, text, and sonification (BTS). This comprehensive approach enhances the accessibility of data visualization and encourages a multi-modal exploration on visualization. Check out the current build: [maidr Demo](https://xability.github.io/maidr/galleries/index.html). You may also clone or download the GitHub repo, navigate to the ./user_study_pilot folder, and open any of the html files in your browser.
 
 ## Table of Contents
 

package/dist/maidr.js

@@ -76,6 +76,26 @@ class Constants {
   globalMinMax = true;
   ariaMode = 'assertive'; // assertive (default) / polite
 
+  userSettingsKeys = [
+    'vol',
+    'autoPlayRate',
+    'brailleDisplayLength',
+    'colorSelected',
+    'MIN_FREQUENCY',
+    'MAX_FREQUENCY',
+    'keypressInterval',
+    'ariaMode',
+    'openAIAuthKey',
+    'geminiAuthKey',
+    'skillLevel',
+    'skillLevelOther',
+    'LLMModel',
+    'LLMPreferences',
+    'LLMOpenAiMulti',
+    'LLMGeminiMulti',
+    'autoInitLLM',
+  ];
+
   // LLM settings
   openAIAuthKey = null; // OpenAI authentication key, set in menu
   geminiAuthKey = null; // Gemini authentication key, set in menu
@@ -389,7 +409,9 @@ class Menu {
                                     <tr>
                                         <td>Open GenAI Chat</td>
                                         <td>${
-                                          constants.control
+                                          constants.isMac
+                                            ? constants.alt
+                                            : constants.control
                                         } + Shift + ?</td>
                                     </tr>
                                     <tr>
@@ -454,12 +476,12 @@ class Menu {
                                     <option value="basic">Basic</option>
                                     <option value="intermediate">Intermediate</option>
                                     <option value="expert">Expert</option>
-                                    <option value="other">other</option>
+                                    <option value="other">Other: describe in your own words</option>
                                 </select>
                                 <label for="skill_level">Level of skill in statistical charts</label>
                             </p>
                             <p id="skill_level_other_container" class="hidden"><input type="text" placeholder="Very basic" id="skill_level_other"> <label for="skill_level_other">Describe your level of skill in statistical charts</label></p>
-                            <p><label for="LLM_preferences">LLM Preferences</label></p>
+                            <p><label for="LLM_preferences">Custom instructions for the chat response</label></p>
                             <p><textarea id="LLM_preferences" rows="4" cols="50" placeholder="I'm a stats undergrad and work with Python. I prefer a casual tone, and favor information accuracy over creative description; just the facts please!"></textarea></p>
                         </div>
                     </div>
@@ -605,13 +627,22 @@ class Menu {
 
     // trigger notification that LLM will be reset
     // this is done on change of LLM model, multi settings, or skill level
-    constants.events.push([
-      document.getElementById('LLM_model'),
-      'change',
-      function (e) {
-        menu.NotifyOfLLMReset();
-      },
-    ]);
+    let LLMResetIds = [
+      'LLM_model',
+      'openai_multi',
+      'gemini_multi',
+      'skill_level',
+      'LLM_preferences',
+    ];
+    for (let i = 0; i < LLMResetIds.length; i++) {
+      constants.events.push([
+        document.getElementById(LLMResetIds[i]),
+        'change',
+        function (e) {
+          menu.NotifyOfLLMReset();
+        },
+      ]);
+    }
   }
 
   /**
@@ -855,6 +886,13 @@ class Menu {
     ) {
       shouldReset = true;
     }
+    if (
+      !shouldReset &&
+      constants.LLMPreferences !=
+        document.getElementById('LLM_preferences').value
+    ) {
+      shouldReset = true;
+    }
     if (
       !shouldReset &&
       constants.LLMModel != document.getElementById('LLM_model').value
@@ -882,24 +920,21 @@ class Menu {
    */
   SaveDataToLocalStorage() {
     let data = {};
-    data.vol = constants.vol;
-    data.autoPlayRate = constants.autoPlayRate;
-    data.brailleDisplayLength = constants.brailleDisplayLength;
-    data.colorSelected = constants.colorSelected;
-    data.MIN_FREQUENCY = constants.MIN_FREQUENCY;
-    data.MAX_FREQUENCY = constants.MAX_FREQUENCY;
-    data.keypressInterval = constants.keypressInterval;
-    data.ariaMode = constants.ariaMode;
-    data.openAIAuthKey = constants.openAIAuthKey;
-    data.geminiAuthKey = constants.geminiAuthKey;
-    data.skillLevel = constants.skillLevel;
-    data.skillLevelOther = constants.skillLevelOther;
-    data.LLMModel = constants.LLMModel;
-    data.LLMPreferences = constants.LLMPreferences;
-    data.LLMOpenAiMulti = constants.LLMOpenAiMulti;
-    data.LLMGeminiMulti = constants.LLMGeminiMulti;
-    data.autoInitLLM = constants.autoInitLLM;
+    for (let i = 0; i < constants.userSettingsKeys.length; i++) {
+      data[constants.userSettingsKeys[i]] =
+        constants[constants.userSettingsKeys[i]];
+    }
     localStorage.setItem('settings_data', JSON.stringify(data));
+
+    // also save to tracking if we're doing that
+    if (constants.isTracking) {
+      // but not auth keys
+      data.openAIAuthKey = 'hidden';
+      data.geminiAuthKey = 'hidden';
+      // and need a timestamp
+      data.timestamp = new Date().toISOString();
+      tracker.SetData('settings', data);
+    }
   }
   /**
    * Loads data from local storage and updates the constants object with the retrieved values, to be loaded into the menu
@@ -907,23 +942,10 @@ class Menu {
   LoadDataFromLocalStorage() {
     let data = JSON.parse(localStorage.getItem('settings_data'));
     if (data) {
-      constants.vol = data.vol;
-      constants.autoPlayRate = data.autoPlayRate;
-      constants.brailleDisplayLength = data.brailleDisplayLength;
-      constants.colorSelected = data.colorSelected;
-      constants.MIN_FREQUENCY = data.MIN_FREQUENCY;
-      constants.MAX_FREQUENCY = data.MAX_FREQUENCY;
-      constants.keypressInterval = data.keypressInterval;
-      constants.ariaMode = data.ariaMode;
-      constants.openAIAuthKey = data.openAIAuthKey;
-      constants.geminiAuthKey = data.geminiAuthKey;
-      constants.skillLevel = data.skillLevel;
-      constants.skillLevelOther = data.skillLevelOther;
-      constants.LLMModel = data.LLMModel ? data.LLMModel : constants.LLMModel;
-      constants.LLMPreferences = data.LLMPreferences;
-      constants.LLMOpenAiMulti = data.LLMOpenAiMulti;
-      constants.LLMGeminiMulti = data.LLMGeminiMulti;
-      constants.autoInitLLM = data.autoInitLLM;
+      for (let i = 0; i < constants.userSettingsKeys.length; i++) {
+        constants[constants.userSettingsKeys[i]] =
+          data[constants.userSettingsKeys[i]];
+      }
     }
     this.PopulateData();
     this.UpdateHtml();
@@ -944,7 +966,16 @@ class ChatLLM {
     this.CreateComponent();
     this.SetEvents();
     if (constants.autoInitLLM) {
-      this.InitChatMessage();
+      // only run if we have API keys set
+      if (
+        (constants.LLMModel == 'openai' && constants.openAIAuthKey) ||
+        (constants.LLMModel == 'gemini' && constants.geminiAuthKey) ||
+        (constants.LLMModel == 'multi' &&
+          constants.openAIAuthKey &&
+          constants.geminiAuthKey)
+      ) {
+        this.InitChatMessage();
+      }
     }
   }
 
@@ -975,11 +1006,10 @@ class ChatLLM {
                             <p><button type="button">What is the title?</button></p>
                             <p><button type="button">What are the high and low values?</button></p>
                             <p><button type="button">What is the general shape of the chart?</button></p>
-                            <p><button type="button" id="more_suggestions">More</button></p>
                           </div>
-                          <div id="more_suggestions_container" class="hidden LLM_suggestions">
+                          <div id="more_suggestions_container" class="LLM_suggestions">
                             <p><button type="button">Please provide the title of this visualization, then provide a description for someone who is blind or low vision. Include general overview of axes and the data at a high-level.</button></p>
-                            <p><button type="button">For the visualization I shared, please provide the following (where applicable): mean, standard deviation, extrema, correlations, relational comparisons like greater than OR lesser than.</button></p>
+                            <p><button type="button">For the visualization I shared, please provide the following (where applicable): mean, standard deviation, extreme, correlations, relational comparisons like greater than OR lesser than.</button></p>
                             <p><button type="button">Based on the visualization shared, address the following: Do you observe any unforeseen trends? If yes, what?  Please convey any complex multi-faceted patterns present. Can you identify any noteworthy exceptions that aren't readily apparent through non-visual methods of analysis?</button></p>
                             <p><button type="button">Provide context to help explain the data depicted in this visualization based on domain-specific insight.</button></p>
                           </div>
@@ -1029,12 +1059,7 @@ class ChatLLM {
       document,
       'keyup',
       function (e) {
-        if (
-          ((e.ctrlKey || e.metaKey) &&
-            e.shiftKey &&
-            (e.key == '?' || e.key == '¿')) ||
-          (e.metaKey && e.altKey && (e.key == '?' || e.key == '¿'))
-        ) {
+        if ((e.key == '?' && (e.ctrlKey || e.metaKey)) || e.key == '¿') {
           chatLLM.Toggle();
         }
       },
@@ -1063,21 +1088,6 @@ class ChatLLM {
     ]);
 
     // ChatLLM suggestion events
-    // the more button
-    constants.events.push([
-      document.getElementById('more_suggestions'),
-      'click',
-      function (e) {
-        document
-          .getElementById('more_suggestions_container')
-          .classList.toggle('hidden');
-        // focus on button right after the more button
-        document
-          .querySelector('#more_suggestions_container > p > button')
-          .focus();
-        document.getElementById('more_suggestions').remove();
-      },
-    ]);
     // actual suggestions:
     let suggestions = document.querySelectorAll(
       '#chatLLM .LLM_suggestions button:not(#more_suggestions)'
@@ -1188,7 +1198,7 @@ class ChatLLM {
       markdown = markdown.replace(/\n{3,}/g, '\n\n');
 
       try {
-        navigator.clipboard.writeText(markdown);
+        navigator.clipboard.writeText(markdown); // note: this fails if you're on the inspector. That's fine as it'll never happen to real users
       } catch (err) {
         console.error('Failed to copy: ', err);
       }
@@ -1358,6 +1368,7 @@ class ChatLLM {
 
       if (data.error) {
         chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.WaitingSound(false);
       } else {
         chatLLM.DisplayChatMessage(LLMName, text);
       }
@@ -1368,10 +1379,12 @@ class ChatLLM {
       } else {
         if (!data.error) {
           data.error = 'Error processing request.';
+          chatLLM.WaitingSound(false);
         }
       }
       if (data.error) {
         chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.WaitingSound(false);
       } else {
         // todo: display actual response
       }
@@ -1472,7 +1485,7 @@ class ChatLLM {
       .catch((error) => {
         chatLLM.WaitingSound(false);
         console.error('Error:', error);
-        chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.DisplayChatMessage('OpenAI', 'Error processing request.', true);
         // also todo: handle errors somehow
       });
   }
@@ -1562,6 +1575,8 @@ class ChatLLM {
       // Process the response
       chatLLM.ProcessLLMResponse(result.response, 'gemini');
     } catch (error) {
+      chatLLM.WaitingSound(false);
+      chatLLM.DisplayChatMessage('Gemini', 'Error processing request.', true);
       console.error('Error in GeminiPrompt:', error);
       throw error; // Rethrow the error for further handling if necessary
     }
@@ -1623,11 +1638,6 @@ class ChatLLM {
   ResetLLM() {
     // clear the main chat history
     document.getElementById('chatLLM_chat_history').innerHTML = '';
-    // unhide the more button
-    document
-      .getElementById('more_suggestions_container')
-      .classList.add('hidden');
-    document.getElementById('more_suggestions').classList.remove('hidden');
 
     // reset the data
     this.requestJson = null;
@@ -2081,7 +2091,7 @@ class Tracker {
   DataSetup() {
     let prevData = this.GetTrackerData();
     if (prevData) {
-      // good to go already, do nothing
+      // good to go already, do nothing, but make sure we have our containers
     } else {
       let data = {};
       data.userAgent = Object.assign(navigator.userAgent);
@@ -2089,8 +2099,10 @@ class Tracker {
       data.language = Object.assign(navigator.language);
       data.platform = Object.assign(navigator.platform);
       data.events = [];
+      data.settings = [];
 
       this.SaveTrackerData(data);
+      this.SaveSettings();
     }
   }
 
@@ -2137,6 +2149,17 @@ class Tracker {
     this.DataSetup();
   }
 
+  SaveSettings() {
+    // fetch all settings, push to data.settings
+    let settings = JSON.parse(localStorage.getItem('settings_data'));
+    if (settings) {
+      // don't store their auth keys
+      settings.openAIAuthKey = 'hidden';
+      settings.geminiAuthKey = 'hidden';
+      this.SetData('settings', settings);
+    }
+  }
+
   /**
    * Logs an event with various properties to the tracker data.
    * @param {Event} e - The event to log.
@@ -2280,6 +2303,7 @@ class Tracker {
         constants.plotOrientation == 'vert' ? position.x : position.y;
       let sectionPos =
         constants.plotOrientation == 'vert' ? position.y : position.x;
+      let sectionLabel = plot.sections[sectionPos];
 
       if (!this.isUndefinedOrNull(plot.x_group_label)) {
         x_label = plot.x_group_label;
@@ -2289,42 +2313,26 @@ class Tracker {
       }
       if (constants.plotOrientation == 'vert') {
         if (plotPos > -1 && sectionPos > -1) {
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].label)
-          ) {
-            y_tickmark = plot.plotData[plotPos][sectionPos].label;
+          if (!this.isUndefinedOrNull(sectionLabel)) {
+            y_tickmark = sectionLabel;
           }
           if (!this.isUndefinedOrNull(plot.x_labels[position.x])) {
             x_tickmark = plot.x_labels[position.x];
           }
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].values)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].values;
-          } else if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].y)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].y;
+          if (!this.isUndefinedOrNull(plot.plotData[plotPos][sectionLabel])) {
+            value = plot.plotData[plotPos][sectionLabel];
           }
         }
       } else {
         if (plotPos > -1 && sectionPos > -1) {
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].label)
-          ) {
-            x_tickmark = plot.plotData[plotPos][sectionPos].label;
+          if (!this.isUndefinedOrNull(sectionLabel)) {
+            x_tickmark = sectionLabel;
           }
           if (!this.isUndefinedOrNull(plot.y_labels[position.y])) {
             y_tickmark = plot.y_labels[position.y];
           }
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].values)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].values;
-          } else if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].x)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].x;
+          if (!this.isUndefinedOrNull(plot.plotData[plotPos][sectionLabel])) {
+            value = plot.plotData[plotPos][sectionLabel];
           }
         }
       }
@@ -2361,10 +2369,14 @@ class Tracker {
 
   SetData(key, value) {
     let data = this.GetTrackerData();
-    if (key == 'events') {
-      data[key].push(value);
-    } else {
+    let arrayKeys = ['events', 'ChatHistory', 'settings'];
+    if (!arrayKeys.includes(key)) {
       data[key] = value;
+    } else {
+      if (!data[key]) {
+        data[key] = [];
+      }
+      data[key].push(value);
     }
     this.SaveTrackerData(data);
   }