maidr 2.9.2 → 2.11.0

package/README.md

@@ -8,7 +8,7 @@
 
 # maidr: Multimodal Access and Interactive Data Representation
 
-maidr (pronounced as 'mader') is a system for non-visual access and control of statistical plots. It aims to provide an inclusive experience for users with visual impairments by offering multiple modes of interaction: braille, text, and sonification (BTS). This comprehensive approach enhances the accessibility of data visualization and encourages a multi-modal exploration on visualization. Check out the current build: [maidr Demo](https://xability.github.io/maidr/user_study_pilot/intro.html). You may also clone or download the GitHub repo, navigate to the ./user_study_pilot folder, and open any of the html files in your browser.
+maidr (pronounced as 'mader') is a system for non-visual access and control of statistical plots. It aims to provide an inclusive experience for users with visual impairments by offering multiple modes of interaction: braille, text, and sonification (BTS). This comprehensive approach enhances the accessibility of data visualization and encourages a multi-modal exploration on visualization. Check out the current build: [maidr Demo](https://xability.github.io/maidr/galleries/index.html). You may also clone or download the GitHub repo, navigate to the ./user_study_pilot folder, and open any of the html files in your browser.
 
 ## Table of Contents
 

package/dist/maidr.js

@@ -109,6 +109,7 @@ class Constants {
   skillLevelOther = ''; // custom skill level
   autoInitLLM = true; // auto initialize LLM on page load
   verboseText = '';
+  waitingQueue = 0;
 
   // user controls (not exposed to menu, with shortcuts usually)
   showDisplay = 1; // true / false
@@ -462,11 +463,11 @@ class Menu {
                             </p>
                             <p id="openai_auth_key_container" class="multi_container hidden">
                               <span id="openai_multi_container" class="hidden"><input type="checkbox" id="openai_multi" name="openai_multi" aria-label="Use OpenAI in Multi modal mode"></span>
-                              <input type="password" id="openai_auth_key"><button aria-label="Delete OpenAI key" title="Delete OpenAI key" id="delete_openai_key" class="invis_button">&times;</button><label for="openai_auth_key">OpenAI Authentication Key</label>
+                              <input type="text" size="50" id="openai_auth_key"><button aria-label="Delete OpenAI key" title="Delete OpenAI key" id="delete_openai_key" class="invis_button">&times;</button><label for="openai_auth_key">OpenAI Authentication Key</label>
                             </p>
                             <p id="gemini_auth_key_container" class="multi_container hidden">
                               <span id="gemini_multi_container" class="hidden"><input type="checkbox" id="gemini_multi" name="gemini_multi" aria-label="Use Gemini in Multi modal mode"></span>
-                              <input type="password" id="gemini_auth_key"><button aria-label="Delete Gemini key" title="Delete Gemini key" id="delete_gemini_key" class="invis_button">&times;</button><label for="gemini_auth_key">Gemini Authentication Key</label>
+                              <input type="text" size="50" id="gemini_auth_key"><button aria-label="Delete Gemini key" title="Delete Gemini key" id="delete_gemini_key" class="invis_button">&times;</button><label for="gemini_auth_key">Gemini Authentication Key</label>
                             </p>
                             <p><input type="checkbox" ${
                               constants.autoInitLLM ? 'checked' : ''
@@ -476,12 +477,12 @@ class Menu {
                                     <option value="basic">Basic</option>
                                     <option value="intermediate">Intermediate</option>
                                     <option value="expert">Expert</option>
-                                    <option value="other">other</option>
+                                    <option value="other">Other: describe in your own words</option>
                                 </select>
                                 <label for="skill_level">Level of skill in statistical charts</label>
                             </p>
                             <p id="skill_level_other_container" class="hidden"><input type="text" placeholder="Very basic" id="skill_level_other"> <label for="skill_level_other">Describe your level of skill in statistical charts</label></p>
-                            <p><label for="LLM_preferences">LLM Preferences</label></p>
+                            <p><label for="LLM_preferences">Custom instructions for the chat response</label></p>
                             <p><textarea id="LLM_preferences" rows="4" cols="50" placeholder="I'm a stats undergrad and work with Python. I prefer a casual tone, and favor information accuracy over creative description; just the facts please!"></textarea></p>
                         </div>
                     </div>
@@ -966,7 +967,16 @@ class ChatLLM {
     this.CreateComponent();
     this.SetEvents();
     if (constants.autoInitLLM) {
-      this.InitChatMessage();
+      // only run if we have API keys set
+      if (
+        (constants.LLMModel == 'openai' && constants.openAIAuthKey) ||
+        (constants.LLMModel == 'gemini' && constants.geminiAuthKey) ||
+        (constants.LLMModel == 'multi' &&
+          constants.openAIAuthKey &&
+          constants.geminiAuthKey)
+      ) {
+        this.InitChatMessage();
+      }
     }
   }
 
@@ -997,11 +1007,10 @@ class ChatLLM {
                             <p><button type="button">What is the title?</button></p>
                             <p><button type="button">What are the high and low values?</button></p>
                             <p><button type="button">What is the general shape of the chart?</button></p>
-                            <p><button type="button" id="more_suggestions">More</button></p>
                           </div>
-                          <div id="more_suggestions_container" class="hidden LLM_suggestions">
+                          <div id="more_suggestions_container" class="LLM_suggestions">
                             <p><button type="button">Please provide the title of this visualization, then provide a description for someone who is blind or low vision. Include general overview of axes and the data at a high-level.</button></p>
-                            <p><button type="button">For the visualization I shared, please provide the following (where applicable): mean, standard deviation, extrema, correlations, relational comparisons like greater than OR lesser than.</button></p>
+                            <p><button type="button">For the visualization I shared, please provide the following (where applicable): mean, standard deviation, extreme, correlations, relational comparisons like greater than OR lesser than.</button></p>
                             <p><button type="button">Based on the visualization shared, address the following: Do you observe any unforeseen trends? If yes, what?  Please convey any complex multi-faceted patterns present. Can you identify any noteworthy exceptions that aren't readily apparent through non-visual methods of analysis?</button></p>
                             <p><button type="button">Provide context to help explain the data depicted in this visualization based on domain-specific insight.</button></p>
                           </div>
@@ -1080,21 +1089,6 @@ class ChatLLM {
     ]);
 
     // ChatLLM suggestion events
-    // the more button
-    constants.events.push([
-      document.getElementById('more_suggestions'),
-      'click',
-      function (e) {
-        document
-          .getElementById('more_suggestions_container')
-          .classList.toggle('hidden');
-        // focus on button right after the more button
-        document
-          .querySelector('#more_suggestions_container > p > button')
-          .focus();
-        document.getElementById('more_suggestions').remove();
-      },
-    ]);
     // actual suggestions:
     let suggestions = document.querySelectorAll(
       '#chatLLM .LLM_suggestions button:not(#more_suggestions)'
@@ -1205,7 +1199,7 @@ class ChatLLM {
       markdown = markdown.replace(/\n{3,}/g, '\n\n');
 
       try {
-        navigator.clipboard.writeText(markdown);
+        navigator.clipboard.writeText(markdown); // note: this fails if you're on the inspector. That's fine as it'll never happen to real users
       } catch (err) {
         console.error('Failed to copy: ', err);
       }
@@ -1317,22 +1311,40 @@ class ChatLLM {
    * @returns {void}
    */
   WaitingSound(onoff = true) {
-    // clear old intervals and timeouts
-    if (constants.waitingInterval) {
-      // destroy old waiting sound
-      clearInterval(constants.waitingInterval);
-      constants.waitingSound = null;
-    }
-    if (constants.waitingSoundOverride) {
-      clearTimeout(constants.waitingSoundOverride);
-      constants.waitingSoundOverride = null;
+    let delay = 1000;
+    let freq = 440; // a440 babee
+    let inprogressFreq = freq * 2;
+
+    if (onoff) {
+      // if turning on, clear old intervals and timeouts
+      if (constants.waitingInterval) {
+        // destroy old waiting sound
+        clearInterval(constants.waitingInterval);
+        constants.waitingInterval = null;
+      }
+      if (constants.waitingSoundOverride) {
+        clearTimeout(constants.waitingSoundOverride);
+        constants.waitingSoundOverride = null;
+      }
+    } else {
+      // notify user that we're done waiting for this one
+      if (audio && chatLLM.shown) {
+        audio.playOscillator(inprogressFreq, 0.2, 0);
+      }
+
+      // turning off, but do we have more in the queue after this?
+      if (constants.waitingQueue > 1) {
+        // turning off and still have a queue, decrement by 1, and play a new sound
+        constants.waitingQueue--;
+      } else {
+        // no queue, just turn off
+        chatLLM.KillAllWaitingSounds();
+      }
     }
 
-    // assuming we're turning it on, start playing a new waiting sound
+    // turning it on: start playing a new waiting sound
     if (onoff) {
       // create new waiting sound
-      let delay = 1000;
-      let freq = 440; // a440 babee
       constants.waitingInterval = setInterval(function () {
         if (audio && chatLLM.shown) {
           audio.playOscillator(freq, 0.2, 0);
@@ -1341,10 +1353,37 @@ class ChatLLM {
 
       // clear automatically after 30 sec, assuming no response
       constants.waitingSoundOverride = setTimeout(function () {
-        chatLLM.WaitingSound(false);
+        chatLLM.KillAllWaitingSounds();
       }, 30000);
+
+      // set queue for multi
+      if (constants.LLMModel != 'multi') {
+        constants.waitingQueue = 1;
+      } else {
+        constants.waitingQueue = 0;
+        if (constants.LLMGeminiMulti) {
+          constants.waitingQueue++;
+        }
+        if (constants.LLMOpenAiMulti) {
+          constants.waitingQueue++;
+        }
+      }
     }
   }
+  /**
+   * Overrides and kills all waiting sounds for LLM
+   */
+  KillAllWaitingSounds() {
+    if (constants.waitingInterval) {
+      clearInterval(constants.waitingInterval);
+      constants.waitingInterval = null;
+    }
+    if (constants.waitingSoundOverride) {
+      clearTimeout(constants.waitingSoundOverride);
+      constants.waitingSoundOverride = null;
+    }
+    constants.waitingQueue = 0;
+  }
 
   InitChatMessage() {
     // get name from resource
@@ -1362,7 +1401,7 @@ class ChatLLM {
    */
   ProcessLLMResponse(data, model) {
     chatLLM.WaitingSound(false);
-    //console.log('LLM response: ', data);
+    console.log('LLM response: ', data);
     let text = '';
     let LLMName = resources.GetString(model);
 
@@ -1375,6 +1414,7 @@ class ChatLLM {
 
       if (data.error) {
         chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.WaitingSound(false);
       } else {
         chatLLM.DisplayChatMessage(LLMName, text);
       }
@@ -1385,10 +1425,12 @@ class ChatLLM {
       } else {
         if (!data.error) {
           data.error = 'Error processing request.';
+          chatLLM.WaitingSound(false);
         }
       }
       if (data.error) {
         chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.WaitingSound(false);
       } else {
         // todo: display actual response
       }
@@ -1489,7 +1531,7 @@ class ChatLLM {
       .catch((error) => {
         chatLLM.WaitingSound(false);
         console.error('Error:', error);
-        chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.DisplayChatMessage('OpenAI', 'Error processing request.', true);
         // also todo: handle errors somehow
       });
   }
@@ -1541,6 +1583,7 @@ class ChatLLM {
   }
 
   async GeminiPrompt(text, imgBase64 = null) {
+    // https://ai.google.dev/docs/gemini_api_overview#node.js
     try {
       // Save the image for next time
       if (imgBase64 == null) {
@@ -1579,6 +1622,8 @@ class ChatLLM {
       // Process the response
       chatLLM.ProcessLLMResponse(result.response, 'gemini');
     } catch (error) {
+      chatLLM.WaitingSound(false);
+      chatLLM.DisplayChatMessage('Gemini', 'Error processing request.', true);
       console.error('Error in GeminiPrompt:', error);
       throw error; // Rethrow the error for further handling if necessary
     }
@@ -1609,9 +1654,11 @@ class ChatLLM {
     let html = `
       <div class="chatLLM_message ${
         user == 'User' ? 'chatLLM_message_self' : 'chatLLM_message_other'
-      }">
-        <${hLevel} class="chatLLM_message_user">${user}</${hLevel}>
-        <p class="chatLLM_message_text">${text}</p>
+      }">`;
+    if (text != resources.GetString('processing')) {
+      html += `<${hLevel} class="chatLLM_message_user">${user}</${hLevel}>`;
+    }
+    html += `<p class="chatLLM_message_text">${text}</p>
       </div>
     `;
     // add a copy button to actual messages
@@ -1640,11 +1687,6 @@ class ChatLLM {
   ResetLLM() {
     // clear the main chat history
     document.getElementById('chatLLM_chat_history').innerHTML = '';
-    // unhide the more button
-    document
-      .getElementById('more_suggestions_container')
-      .classList.add('hidden');
-    document.getElementById('more_suggestions').classList.remove('hidden');
 
     // reset the data
     this.requestJson = null;
@@ -2159,10 +2201,12 @@ class Tracker {
   SaveSettings() {
     // fetch all settings, push to data.settings
     let settings = JSON.parse(localStorage.getItem('settings_data'));
-    // don't store their auth keys
-    settings.openAIAuthKey = 'hidden';
-    settings.geminiAuthKey = 'hidden';
-    this.SetData('settings', settings);
+    if (settings) {
+      // don't store their auth keys
+      settings.openAIAuthKey = 'hidden';
+      settings.geminiAuthKey = 'hidden';
+      this.SetData('settings', settings);
+    }
   }
 
   /**
@@ -2308,6 +2352,7 @@ class Tracker {
         constants.plotOrientation == 'vert' ? position.x : position.y;
       let sectionPos =
         constants.plotOrientation == 'vert' ? position.y : position.x;
+      let sectionLabel = plot.sections[sectionPos];
 
       if (!this.isUndefinedOrNull(plot.x_group_label)) {
         x_label = plot.x_group_label;
@@ -2317,42 +2362,26 @@ class Tracker {
       }
       if (constants.plotOrientation == 'vert') {
         if (plotPos > -1 && sectionPos > -1) {
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].label)
-          ) {
-            y_tickmark = plot.plotData[plotPos][sectionPos].label;
+          if (!this.isUndefinedOrNull(sectionLabel)) {
+            y_tickmark = sectionLabel;
           }
           if (!this.isUndefinedOrNull(plot.x_labels[position.x])) {
             x_tickmark = plot.x_labels[position.x];
           }
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].values)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].values;
-          } else if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].y)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].y;
+          if (!this.isUndefinedOrNull(plot.plotData[plotPos][sectionLabel])) {
+            value = plot.plotData[plotPos][sectionLabel];
           }
         }
       } else {
         if (plotPos > -1 && sectionPos > -1) {
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].label)
-          ) {
-            x_tickmark = plot.plotData[plotPos][sectionPos].label;
+          if (!this.isUndefinedOrNull(sectionLabel)) {
+            x_tickmark = sectionLabel;
           }
           if (!this.isUndefinedOrNull(plot.y_labels[position.y])) {
             y_tickmark = plot.y_labels[position.y];
           }
-          if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].values)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].values;
-          } else if (
-            !this.isUndefinedOrNull(plot.plotData[plotPos][sectionPos].x)
-          ) {
-            value = plot.plotData[plotPos][sectionPos].x;
+          if (!this.isUndefinedOrNull(plot.plotData[plotPos][sectionLabel])) {
+            value = plot.plotData[plotPos][sectionLabel];
           }
         }
       }