maidr 2.21.0 → 2.22.1

package/dist/maidr.js

@@ -406,13 +406,6 @@ class Constants {
    * @memberof AdvancedUserSettings
    */
   canTrack = 0; // 0 / 1, can we track user data
-  /**
-   * Whether or not we're currently tracking user data.
-   * @type {boolean}
-   * @default 1
-   * @memberof AdvancedUserSettings
-   */
-  isTracking = 0;
   /**
    * How are we representing braille? like, is it 1:1 with the chart, or do we do some compression and try to represent as accuratly as we can? Not currently in use.
    * @type {boolean}
@@ -443,12 +436,15 @@ class Constants {
     'ariaMode',
     'openAIAuthKey',
     'geminiAuthKey',
+    'claudeAuthKey',
+    'emailAuthKey',
     'skillLevel',
     'skillLevelOther',
     'LLMModel',
     'LLMPreferences',
     'LLMOpenAiMulti',
     'LLMGeminiMulti',
+    'LLMModels',
     'autoInitLLM',
   ];
 
@@ -498,6 +494,13 @@ class Constants {
    * @memberof LLMSettings
    */
   LLMModel = 'openai';
+  /**
+   * Current LLM model in use. Can be 'openai' (default) or 'gemini' or 'claude'. More to be added.
+   * @type {("openai"|"gemini"|"claude")}
+   * @default 'openai'
+   * @memberof LLMSettings
+   */
+  LLMModels = { openai: true };
   /**
    * The default system message for the LLM. Helps the LLM understand the context of the chart and its role.
    * @type {string}
@@ -832,6 +835,7 @@ class Resources {
         empty: 'Empty',
         openai: 'OpenAI Vision',
         gemini: 'Gemini Pro Vision',
+        claude: 'Claude',
         multi: 'Multiple AI',
         processing: 'Processing Chart...',
       },
@@ -1011,8 +1015,7 @@ class Menu {
                                 constants.ariaMode == 'polite' ? 'checked' : ''
                               }><label for="aria_mode_polite">Polite</label></p>
                               </fieldset></div>
-                            <h5 class="modal-title">LLM Settings</h5>
-                            <p>
+                            <p class="hidden">
                                 <select id="LLM_model">
                                     <option value="openai">OpenAI Vision</option>
                                     <option value="gemini">Gemini Pro Vision</option>
@@ -1020,6 +1023,21 @@ class Menu {
                                 </select>
                                 <label for="LLM_model">LLM Model</label>
                             </p>
+                            <h5 class="modal-title">LLM Settings</h5>
+                            <p>
+                              <fieldset>
+                                <legend>LLM Models (select up to 2)</legend>
+                                <p><input type="checkbox" id="LLM_model_openai" name="LLM_model" value="openai"><label for="LLM_model_openai">OpenAI Vision</label></p>
+                                <p><input type="checkbox" id="LLM_model_gemini" name="LLM_model" value="gemini"><label for="LLM_model_gemini">Gemini Pro Vision</label></p>
+                                <p><input type="checkbox" id="LLM_model_claude" name="LLM_model" value="claude"><label for="LLM_model_claude">Claude</label></p>
+                              </fieldset>
+                            </p>
+                            <p id="email_auth_key_container" class="multi_container">
+                              <input type="email" size="50" id="email_auth_key" aria-label="Enter your email address">
+                              <button aria-label="Delete Email Address" title="Delete Email Address" id="delete_email_key" class="invis_button">&times;</button>
+                              <label for="gemini_auth_key">Email Authentication</label>
+                              <button type="button" id="verify">Verify</button>
+                            </p>
                             <p id="openai_auth_key_container" class="multi_container hidden">
                               <span id="openai_multi_container" class="hidden"><input type="checkbox" id="openai_multi" name="openai_multi" aria-label="Use OpenAI in Multi modal mode"></span>
                               <input type="password" size="50" id="openai_auth_key"><button aria-label="Delete OpenAI key" title="Delete OpenAI key" id="delete_openai_key" class="invis_button">&times;</button><label for="openai_auth_key">OpenAI Authentication Key</label>
@@ -1027,10 +1045,14 @@ class Menu {
                             <p id="gemini_auth_key_container" class="multi_container hidden">
                               <span id="gemini_multi_container" class="hidden"><input type="checkbox" id="gemini_multi" name="gemini_multi" aria-label="Use Gemini in Multi modal mode"></span>
                               <input type="password" size="50" id="gemini_auth_key"><button aria-label="Delete Gemini key" title="Delete Gemini key" id="delete_gemini_key" class="invis_button">&times;</button><label for="gemini_auth_key">Gemini Authentication Key</label>
+                            </p>
+                             <p id="claude_auth_key_container" class="multi_container hidden">
+                              <span id="claude_multi_container" class="hidden"><input type="checkbox" id="claude_multi" name="claude_multi" aria-label="Use Claude in Multi modal mode"></span>
+                              <input type="password" size="50" id="claude_auth_key"><button aria-label="Delete Claude key" title="Delete Claude key" id="delete_claude_key" class="invis_button">&times;</button><label for="claude_auth_key">Claude Authentication Key</label>
                             </p>
                             <p><input type="checkbox" ${
                               constants.autoInitLLM ? 'checked' : ''
-                            } id="init_llm_on_load" name="init_llm_on_load"><label for="init_llm_on_load">Start first LLM chat chart load</label></p>
+                            } id="init_llm_on_load" name="init_llm_on_load"><label for="init_llm_on_load">Start LLM right away</label></p>
                             <p>
                                 <select id="skill_level">
                                     <option value="basic">Basic</option>
@@ -1088,6 +1110,13 @@ class Menu {
         menu.Toggle(false);
       },
     ]);
+    constants.events.push([
+      document.getElementById('verify'),
+      'click',
+      function (e) {
+        menu.VerifyEmail();
+      },
+    ]);
     constants.events.push([
       document.getElementById('menu'),
       'keyup',
@@ -1170,6 +1199,54 @@ class Menu {
       },
     ]);
 
+    constants.events.push([
+      document.getElementById('LLM_model_openai'),
+      'change',
+      function (e) {
+        if (e.target.checked) {
+          document
+            .getElementById('openai_auth_key_container')
+            .classList.remove('hidden');
+        } else {
+          document
+            .getElementById('openai_auth_key_container')
+            .classList.add('hidden');
+        }
+      },
+    ]);
+
+    constants.events.push([
+      document.getElementById('LLM_model_gemini'),
+      'change',
+      function (e) {
+        if (e.target.checked) {
+          document
+            .getElementById('gemini_auth_key_container')
+            .classList.remove('hidden');
+        } else {
+          document
+            .getElementById('gemini_auth_key_container')
+            .classList.add('hidden');
+        }
+      },
+    ]);
+
+    constants.events.push([
+      document.getElementById('LLM_model_claude'),
+      'change',
+      function (e) {
+        // if (e.target.checked) {
+        document
+          .getElementById('claude_auth_key_container')
+          .classList.add('hidden');
+        // } else {
+        //   document
+        //     .getElementById('claude_auth_key_container')
+        //     .classList.add('hidden');
+        // }
+      },
+    ]);
+
     // Skill level other events
     constants.events.push([
       document.getElementById('skill_level'),
@@ -1205,6 +1282,20 @@ class Menu {
         },
       ]);
     }
+
+    // Limit selections to 2 AI models
+    const llmCheckboxes = document.querySelectorAll('input[name="LLM_model"]');
+    llmCheckboxes.forEach((checkbox) => {
+      checkbox.addEventListener('change', () => {
+        const checked = document.querySelectorAll(
+          'input[name="LLM_model"]:checked'
+        );
+        if (checked.length > 2) {
+          checkbox.checked = false;
+          alert('You can select up to 2 AI models.');
+        }
+      });
+    });
   }
 
   /**
@@ -1276,16 +1367,22 @@ class Menu {
       document.getElementById('openai_auth_key').value =
         constants.openAIAuthKey;
     }
+    if (typeof constants.emailAuthKey == 'string') {
+      document.getElementById('email_auth_key').value = constants.emailAuthKey;
+    }
     if (typeof constants.geminiAuthKey == 'string') {
       document.getElementById('gemini_auth_key').value =
         constants.geminiAuthKey;
     }
+    if (typeof constants.claudeAuthKey == 'string') {
+      document.getElementById('claude_auth_key').value =
+        constants.claudeAuthKey;
+    }
     document.getElementById('skill_level').value = constants.skillLevel;
     if (constants.skillLevelOther) {
       document.getElementById('skill_level_other').value =
         constants.skillLevelOther;
     }
-    document.getElementById('LLM_model').value = constants.LLMModel;
 
     // aria mode
     if (constants.ariaMode == 'assertive') {
@@ -1295,44 +1392,18 @@ class Menu {
       document.getElementById('aria_mode_polite').checked = true;
       document.getElementById('aria_mode_assertive').checked = false;
     }
-    // hide either openai or gemini auth key field
-    if (constants.LLMModel == 'openai') {
-      document
-        .getElementById('openai_auth_key_container')
-        .classList.remove('hidden');
-      document
-        .getElementById('gemini_auth_key_container')
-        .classList.add('hidden');
-    } else if (constants.LLMModel == 'gemini') {
-      document
-        .getElementById('openai_auth_key_container')
-        .classList.add('hidden');
-      document
-        .getElementById('gemini_auth_key_container')
-        .classList.remove('hidden');
-    } else if (constants.LLMModel == 'multi') {
-      // multi LLM mode
-      document
-        .getElementById('openai_auth_key_container')
-        .classList.remove('hidden');
-      document
-        .getElementById('gemini_auth_key_container')
-        .classList.remove('hidden');
-      document
-        .getElementById('openai_multi_container')
-        .classList.remove('hidden');
+
+    for (let model in constants.LLMModels) {
+      document.getElementById(`LLM_model_${model}`).checked = true;
+
       document
-        .getElementById('gemini_multi_container')
+        .getElementById(`${model}_auth_key_container`)
         .classList.remove('hidden');
-      document.getElementById('openai_multi').checked = false;
-      if (constants.LLMOpenAiMulti) {
-        document.getElementById('openai_multi').checked = true;
-      }
-      document.getElementById('gemini_multi').checked = false;
-      if (constants.LLMGeminiMulti) {
-        document.getElementById('gemini_multi').checked = true;
-      }
     }
+    document
+      .getElementById(`claude_auth_key_container`)
+      .classList.add('hidden');
+
     // skill level other
     if (constants.skillLevel == 'other') {
       document
@@ -1368,10 +1439,22 @@ class Menu {
 
     constants.openAIAuthKey = document.getElementById('openai_auth_key').value;
     constants.geminiAuthKey = document.getElementById('gemini_auth_key').value;
+    constants.claudeAuthKey = document.getElementById('claude_auth_key').value;
+    constants.emailAuthKey = document.getElementById('email_auth_key').value;
     constants.skillLevel = document.getElementById('skill_level').value;
     constants.skillLevelOther =
       document.getElementById('skill_level_other').value;
-    constants.LLMModel = document.getElementById('LLM_model').value;
+    // constants.LLMModel = document.getElementById('LLM_model').value;
+
+    const llmCheckboxes = document.querySelectorAll('input[name="LLM_model"]');
+    llmCheckboxes.forEach((checkbox) => {
+      if (checkbox.checked) {
+        constants.LLMModels[checkbox.value] = true;
+      } else {
+        delete constants.LLMModels[checkbox.value];
+      }
+    });
+
     constants.LLMPreferences = document.getElementById('LLM_preferences').value;
     constants.LLMOpenAiMulti = document.getElementById('openai_multi').checked;
     constants.LLMGeminiMulti = document.getElementById('gemini_multi').checked;
@@ -1394,6 +1477,41 @@ class Menu {
     }
   }
 
+  VerifyEmail() {
+    let email = document.getElementById('email_auth_key').value;
+    if (email && email.indexOf('@') !== -1) {
+      let url = `https://maidr-service.azurewebsites.net/api/send_email?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D`;
+
+      let requestJson = {
+        email: email,
+      };
+
+      fetch(url, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authentication: constants.emailAuthKey,
+        },
+        body: JSON.stringify(requestJson),
+      })
+        .then((response) => response.json())
+        .then((data) => {
+          if (data && data.success) {
+            alert('Link sent to email address: ' + email);
+          } else {
+            console.log(data);
+            alert(data.data);
+          }
+        })
+        .catch((error) => {
+          console.log(error);
+          alert(error.data);
+        });
+    } else {
+      alert('Please enter a valid email address.');
+    }
+  }
+
   /**
    * Sets the aria attributes on the HTML elements in the menu
    * @returns {void}
@@ -1475,14 +1593,16 @@ class Menu {
     ) {
       shouldReset = true;
     }
-    if (
-      !shouldReset &&
-      (constants.LLMOpenAiMulti !=
-        document.getElementById('openai_multi').checked ||
-        constants.LLMGeminiMulti !=
-          document.getElementById('gemini_multi').checked)
-    ) {
-      shouldReset = true;
+
+    // check if LLMModels have changed
+    let llmCheckboxes = document.querySelectorAll('input[name="LLM_model"]');
+    for (let i = 0; i < llmCheckboxes.length; i++) {
+      if (
+        !shouldReset &&
+        constants.LLMModels[llmCheckboxes[i].value] != llmCheckboxes[i].checked
+      ) {
+        shouldReset = true;
+      }
     }
 
     return shouldReset;
@@ -1501,10 +1621,11 @@ class Menu {
     localStorage.setItem('settings_data', JSON.stringify(data));
 
     // also save to tracking if we're doing that
-    if (constants.isTracking) {
+    if (constants.canTrack) {
       // but not auth keys
       data.openAIAuthKey = 'hidden';
       data.geminiAuthKey = 'hidden';
+      data.claudeAuthKey = 'hidden';
       // and need a timestamp
       data.timestamp = new Date().toISOString();
       tracker.SetData('settings', data);
@@ -1517,9 +1638,10 @@ class Menu {
     let data = JSON.parse(localStorage.getItem('settings_data'));
     if (data) {
       for (let i = 0; i < constants.userSettingsKeys.length; i++) {
-        constants[constants.userSettingsKeys[i]] =
-          data[constants.userSettingsKeys[i]] ||
-          constants[constants.userSettingsKeys[i]];
+        const key = constants.userSettingsKeys[i];
+        if (key in data) {
+          constants[key] = data[key];
+        }
       }
     }
     this.PopulateData();
@@ -1541,11 +1663,9 @@ class ChatLLM {
     if (constants.autoInitLLM) {
       // only run if we have API keys set
       if (
-        (constants.LLMModel == 'openai' && constants.openAIAuthKey) ||
-        (constants.LLMModel == 'gemini' && constants.geminiAuthKey) ||
-        (constants.LLMModel == 'multi' &&
-          constants.openAIAuthKey &&
-          constants.geminiAuthKey)
+        ('gemini' in constants.LLMModels && constants.geminiAuthKey) ||
+        ('openai' in constants.LLMModels && constants.openAIAuthKey) ||
+        ('claude' in constants.LLMModels && constants.claudeAuthKey)
       ) {
         this.InitChatMessage();
       }
@@ -1686,6 +1806,13 @@ class ChatLLM {
         document.getElementById('openai_auth_key').value = '';
       },
     ]);
+    constants.events.push([
+      document.getElementById('delete_email_key'),
+      'click',
+      function (e) {
+        document.getElementById('email_auth_key').value = '';
+      },
+    ]);
     constants.events.push([
       document.getElementById('delete_gemini_key'),
       'click',
@@ -1853,7 +1980,7 @@ class ChatLLM {
 
     // if this is the user's first message (or we're gemini, in which case we need to send every time), prepend prompt with user position
     if (
-      (this.firstOpen || constants.LLMModel == 'gemini') &&
+      (this.firstOpen || 'gemini' in constants.LLMModels) &&
       !firsttime &&
       constants.verboseText.length > 0
     ) {
@@ -1871,17 +1998,36 @@ class ChatLLM {
       this.WaitingSound(true);
     }
 
-    if (constants.LLMOpenAiMulti || constants.LLMModel == 'openai') {
+    if ('openai' in constants.LLMModels) {
       if (firsttime) {
         img = await this.ConvertSVGtoJPG(singleMaidr.id, 'openai');
       }
-      chatLLM.OpenAIPrompt(text, img);
+      if (constants.openAIAuthKey) {
+        chatLLM.OpenAIPrompt(text, img);
+      } else {
+        chatLLM.OpenAIPromptAPI(text, img);
+      }
     }
-    if (constants.LLMGeminiMulti || constants.LLMModel == 'gemini') {
+    if ('gemini' in constants.LLMModels) {
       if (firsttime) {
         img = await this.ConvertSVGtoJPG(singleMaidr.id, 'gemini');
       }
-      chatLLM.GeminiPrompt(text, img);
+      if (constants.geminiAuthKey) {
+        chatLLM.GeminiPrompt(text, img);
+      } else {
+        chatLLM.GeminiPromptAPI(text, img);
+      }
+    }
+
+    if ('claude' in constants.LLMModels) {
+      if (firsttime) {
+        img = await this.ConvertSVGtoJPG(singleMaidr.id, 'claude');
+      }
+      if (constants.claudeAuthKey) {
+        chatLLM.ClaudePrompt(text, img);
+      } else {
+        chatLLM.ClaudePromptAPI(text, img);
+      }
     }
   }
 
@@ -1969,7 +2115,7 @@ class ChatLLM {
   }
 
   InitChatMessage() {
-    // get name from resource
+    // get name from resource]
     let LLMName = resources.GetString(constants.LLMModel);
     this.firstTime = false;
     this.DisplayChatMessage(LLMName, resources.GetString('processing'), true);
@@ -1984,7 +2130,6 @@ class ChatLLM {
    */
   ProcessLLMResponse(data, model) {
     chatLLM.WaitingSound(false);
-    //console.log('LLM response: ', data);
     let text = '';
     let LLMName = resources.GetString(model);
 
@@ -2018,9 +2163,20 @@ class ChatLLM {
         // todo: display actual response
       }
     }
+    if (model == 'claude') {
+      console.log('Claude response: ', data);
+      if (data.text()) {
+        text = data.text();
+        chatLLM.DisplayChatMessage(LLMName, text);
+      }
+      if (data.error) {
+        chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true);
+        chatLLM.WaitingSound(false);
+      }
+    }
 
     // if we're tracking, log the data
-    if (constants.isTracking) {
+    if (constants.canTrack) {
       let chatHist = chatLLM.CopyChatHistory();
       tracker.SetData('ChatHistory', chatHist);
     }
@@ -2086,6 +2242,96 @@ class ChatLLM {
     return responseText;
   }
 
+  ClaudeJson(text, img = null) {
+    const anthropicVersion = 'vertex-2023-10-16';
+    const maxTokens = 256;
+
+    const payload = {
+      anthropic_version: anthropicVersion,
+      max_tokens: maxTokens,
+      messages: [],
+    };
+
+    // Construct the user message object
+    const userMessage = {
+      role: 'user',
+      content: [],
+    };
+
+    // Add the image content if provided
+    if (img) {
+      userMessage.content.push(
+        {
+          type: 'image',
+          source: {
+            type: 'base64',
+            media_type: 'image/jpeg', // Update if other formats are supported
+            data: img,
+          },
+        },
+        {
+          type: 'text',
+          text: text,
+        }
+      );
+    } else {
+      // Add only the text content if no image is provided
+      userMessage.content.push({
+        type: 'text',
+        text: text,
+      });
+    }
+
+    // Add the user message to the messages array
+    payload.messages.push(userMessage);
+
+    return payload;
+  }
+
+  ClaudePromptAPI(text, imgBase64 = null) {
+    console.log('Claude prompt API');
+    let url =
+      'https://maidr-service.azurewebsites.net/api/claude?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D';
+
+    // Create the prompt
+    let prompt = constants.LLMSystemMessage;
+    if (constants.LLMPreferences) {
+      prompt += constants.LLMPreferences;
+    }
+    prompt += '\n\n' + text; // Use the text parameter as the prompt
+
+    if (imgBase64 == null) {
+      imgBase64 = constants.LLMImage;
+    } else {
+      constants.LLMImage = imgBase64;
+    }
+    constants.LLMImage = imgBase64;
+
+    let requestJson = chatLLM.ClaudeJson(prompt, imgBase64);
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authentication: constants.emailAuthKey,
+      },
+      body: JSON.stringify(requestJson),
+    })
+      .then((response) => response.json())
+      .then((data) => {
+        data.text = function () {
+          return data.content[0].text;
+        };
+        chatLLM.ProcessLLMResponse(data, 'claude');
+      })
+      .catch((error) => {
+        chatLLM.WaitingSound(false);
+        console.error('Error:', error);
+        chatLLM.DisplayChatMessage('Claude', 'Error processing request.', true);
+        // also todo: handle errors somehow
+      });
+  }
+
   /**
    * Gets running prompt info, appends the latest request, and packages it into a JSON object for the LLM.
    * @function
@@ -2119,6 +2365,35 @@ class ChatLLM {
         // also todo: handle errors somehow
       });
   }
+
+  OpenAIPromptAPI(text, img = null) {
+    // request init
+    let url =
+      'https://maidr-service.azurewebsites.net/api/openai?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D';
+    let auth = constants.openAIAuthKey;
+    let requestJson = chatLLM.OpenAIJson(text, img);
+    console.log('LLM request: ', requestJson);
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authentication: constants.emailAuthKey,
+      },
+      body: JSON.stringify(requestJson),
+    })
+      .then((response) => response.json())
+      .then((data) => {
+        chatLLM.ProcessLLMResponse(data, 'openai');
+      })
+      .catch((error) => {
+        chatLLM.WaitingSound(false);
+        console.error('Error:', error);
+        chatLLM.DisplayChatMessage('OpenAI', 'Error processing request.', true);
+        // also todo: handle errors somehow
+      });
+  }
+
   OpenAIJson(text, img = null) {
     let sysMessage = constants.LLMSystemMessage;
     let backupMessage =
@@ -2167,6 +2442,110 @@ class ChatLLM {
     return this.requestJson;
   }
 
+  GeminiJson(text, img = null) {
+    let sysMessage = constants.LLMSystemMessage;
+    let backupMessage =
+      'Describe ' + singleMaidr.type + ' charts to a blind person';
+
+    let payload = {
+      generationConfig: {},
+      safetySettings: [],
+      contents: [],
+    };
+
+    // System message as the initial "role" and "text" content for context
+    let sysContent = {
+      role: 'user',
+      parts: [
+        {
+          text: sysMessage || backupMessage, // Fallback if sysMessage is unavailable
+        },
+      ],
+    };
+
+    // Add preferences if available
+    if (constants.LLMPreferences) {
+      sysContent.parts.push({
+        text: constants.LLMPreferences,
+      });
+    }
+
+    payload.contents.push(sysContent);
+
+    // Add user input content, including image if available
+    let userContent = {
+      role: 'user',
+      parts: [],
+    };
+
+    if (img) {
+      // If there’s an image, add both text and image data
+      userContent.parts.push(
+        {
+          text: text,
+        },
+        {
+          inlineData: {
+            data: img, // Expecting base64-encoded image data
+            mimeType: 'image/png', // Adjust if different image formats are possible
+          },
+        }
+      );
+    } else {
+      // If no image, only add the text
+      userContent.parts.push({
+        text: text,
+      });
+    }
+
+    // Add user content to the contents array
+    payload.contents.push(userContent);
+
+    return payload;
+  }
+
+  async GeminiPromptAPI(text, imgBase64 = null) {
+    let url =
+      'https://maidr-service.azurewebsites.net/api/gemini?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D';
+
+    // Create the prompt
+    let prompt = constants.LLMSystemMessage;
+    if (constants.LLMPreferences) {
+      prompt += constants.LLMPreferences;
+    }
+    prompt += '\n\n' + text; // Use the text parameter as the prompt
+
+    if (imgBase64 == null) {
+      imgBase64 = constants.LLMImage;
+    } else {
+      constants.LLMImage = imgBase64;
+    }
+    constants.LLMImage = imgBase64;
+
+    let requestJson = chatLLM.GeminiJson(prompt, imgBase64);
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authentication: constants.emailAuthKey,
+      },
+      body: JSON.stringify(requestJson),
+    });
+    if (response.ok) {
+      const responseJson = await response.json();
+      responseJson.text = () => {
+        return responseJson.candidates[0].content.parts[0].text;
+      };
+      chatLLM.ProcessLLMResponse(responseJson, 'gemini');
+    } else {
+      chatLLM.WaitingSound(false);
+      console.error('Error:', error);
+      chatLLM.DisplayChatMessage('OpenAI', 'Error processing request.', true);
+      // also todo: handle errors somehow
+    }
+  }
+
   async GeminiPrompt(text, imgBase64 = null) {
     // https://ai.google.dev/docs/gemini_api_overview#node.js
     try {
@@ -2366,7 +2745,7 @@ class ChatLLM {
         var jpegData = canvas.toDataURL('image/jpeg', 0.9); // 0.9 is the quality parameter
         if (model == 'openai') {
           resolve(jpegData);
-        } else if (model == 'gemini') {
+        } else if (model == 'gemini' || model == 'claude') {
           let base64Data = jpegData.split(',')[1];
           resolve(base64Data);
           //resolve(jpegData);
@@ -2716,9 +3095,13 @@ class Helper {
  * @class
  */
 class Tracker {
+  // URL
+  logUrl =
+    'https://maidr-service.azurewebsites.net/api/log?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D'; // TODO Replace
+  isLocal = false;
+
   constructor() {
     this.DataSetup();
-    constants.isTracking = true;
   }
 
   /**
@@ -2726,14 +3109,14 @@ class Tracker {
    */
   DataSetup() {
     let prevData = this.GetTrackerData();
-    if (prevData) {
-      // good to go already, do nothing, but make sure we have our containers
-    } else {
+    if (!this.isLocal || !prevData) {
       let data = {};
       data.userAgent = Object.assign(navigator.userAgent);
       data.vendor = Object.assign(navigator.vendor);
       data.language = Object.assign(navigator.language);
       data.platform = Object.assign(navigator.platform);
+      data.geolocation = Object.assign(navigator.geolocation);
+      data.log_type = 'system_data';
       data.events = [];
       data.settings = [];
 
@@ -2758,8 +3141,33 @@ class Tracker {
    * Saves the tracker data to local storage.
    * @param {Object} data - The data to be saved.
    */
-  SaveTrackerData(data) {
-    localStorage.setItem(constants.project_id, JSON.stringify(data));
+  async SaveTrackerData(data) {
+    console.log('about to save data', data);
+    if (this.isLocal) {
+      localStorage.setItem(constants.project_id, JSON.stringify(data));
+    } else {
+      // test this first
+      try {
+        const response = await fetch(this.logUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(data),
+        });
+
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+
+        const result = await response.json();
+        console.log('Data saved successfully:', result);
+        return result;
+      } catch (error) {
+        console.error('Error saving data:', error);
+        return null;
+      }
+    }
   }
 
   /**
@@ -2792,6 +3200,10 @@ class Tracker {
       // don't store their auth keys
       settings.openAIAuthKey = 'hidden';
       settings.geminiAuthKey = 'hidden';
+      if (constants.emailAuthKey) {
+        settings.username = constants.emailAuthKey;
+      }
+      settings;
       this.SetData('settings', settings);
     }
   }
@@ -2810,6 +3222,9 @@ class Tracker {
     eventToLog.altKey = Object.assign(e.altKey);
     eventToLog.ctrlKey = Object.assign(e.ctrlKey);
     eventToLog.shiftKey = Object.assign(e.shiftKey);
+    if (constants.emailAuthKey) {
+      eventToLog.username = Object.assign(constants.emailAuthKey);
+    }
     if (e.path) {
       eventToLog.focus = Object.assign(e.path[0].tagName);
     }
@@ -3003,18 +3418,29 @@ class Tracker {
     //console.log('logged an event');
   }
 
+  /**
+   * Saves data to the server using a POST request.
+   * @param {Object} logData - The data to be saved.
+   * @returns {Promise<Object>} The result of the save operation.
+   */
+
   SetData(key, value) {
-    let data = this.GetTrackerData();
-    let arrayKeys = ['events', 'ChatHistory', 'settings'];
-    if (!arrayKeys.includes(key)) {
-      data[key] = value;
-    } else {
-      if (!data[key]) {
-        data[key] = [];
+    if (this.isLocal) {
+      let data = this.GetTrackerData();
+      let arrayKeys = ['events', 'ChatHistory', 'settings'];
+      if (!arrayKeys.includes(key)) {
+        data[key] = value;
+      } else {
+        if (!data[key]) {
+          data[key] = [];
+        }
+        data[key].push(value);
       }
-      data[key].push(value);
+      this.SaveTrackerData(data);
+    } else {
+      value['log_type'] = key;
+      this.SaveTrackerData(value);
     }
-    this.SaveTrackerData(data);
   }
 
   /**
@@ -11121,47 +11547,6 @@ document.addEventListener('DOMContentLoaded', function (e) {
 
   // init components like alt text on just the first chart
   CreateChartComponents(firstMaidr, true);
-
-  // events etc for user study page
-  // run tracker stuff only on user study page
-  if (constants.canTrack) {
-    window.tracker = new Tracker();
-    if (document.getElementById('download_data_trigger')) {
-      // we're on the intro page, so enable the download data button
-      document
-        .getElementById('download_data_trigger')
-        .addEventListener('click', function (e) {
-          tracker.DownloadTrackerData();
-        });
-    }
-
-    // general events
-    document.addEventListener('keydown', function (e) {
-      // reset tracking with Ctrl + F5 / command + F5, and Ctrl + Shift + R / command + Shift + R
-      // future todo: this should probably be a button with a confirmation. This is dangerous
-      if (
-        (e.key == 'F5' && (constants.isMac ? e.metaKey : e.ctrlKey)) ||
-        (e.key == 'R' && (constants.isMac ? e.metaKey : e.ctrlKey))
-      ) {
-        e.preventDefault();
-        tracker.Delete();
-        location.reload(true);
-      }
-
-      // main event tracker, built for individual charts
-      if (e.key == 'F10') {
-        tracker.DownloadTrackerData();
-      } else {
-        if (plot) {
-          tracker.LogEvent(e);
-        }
-      }
-
-      // Stuff to only run if we're on a chart (so check if the info div exists?)
-      if (document.getElementById('info')) {
-      }
-    });
-  }
 });
 
 /**
@@ -11181,6 +11566,7 @@ function InitMaidr(thisMaidr) {
     }
     DestroyChartComponents(); // destroy so that we start fresh, in case we've created on the wrong chart
     CreateChartComponents(singleMaidr);
+    InitTracker();
 
     window.menu = new Menu();
     window.chatLLM = new ChatLLM();
@@ -11586,6 +11972,44 @@ function CreateChartComponents(thisMaidr, chartOnly = false) {
   }
 }
 
+function InitTracker() {
+  // events etc for user study page
+  // run tracker stuff only on user study page
+  if (constants.canTrack) {
+    window.tracker = new Tracker();
+    if (document.getElementById('download_data_trigger')) {
+      // we're on the intro page, so enable the download data button
+      document
+        .getElementById('download_data_trigger')
+        .addEventListener('click', function (e) {
+          tracker.DownloadTrackerData();
+        });
+    }
+
+    // general events
+    document.addEventListener('keydown', function (e) {
+      // reset tracking with Ctrl + F5 / command + F5, and Ctrl + Shift + R / command + Shift + R
+      // future todo: this should probably be a button with a confirmation. This is dangerous
+      if (
+        (e.key == 'F5' && (constants.isMac ? e.metaKey : e.ctrlKey)) ||
+        (e.key == 'R' && (constants.isMac ? e.metaKey : e.ctrlKey))
+      ) {
+        e.preventDefault();
+        tracker.Delete();
+        location.reload(true);
+      }
+
+      // main event tracker, built for individual charts
+      if (e.key == 'F10') {
+        tracker.DownloadTrackerData();
+      } else {
+        if (plot) {
+          tracker.LogEvent(e);
+        }
+      }
+    });
+  }
+}
 /**
  * Removes all chart components from the DOM and resets related variables to null.
  * @function