@adminforth/agent 1.43.3 → 1.43.5

package/build.log

@@ -58,5 +58,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
 custom/speech_recognition_frontend/types/
 custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
 
-sent 1,664,048 bytes  received 860 bytes  3,329,816.00 bytes/sec
-total size is 1,660,186  speedup is 1.00
+sent 1,661,194 bytes  received 860 bytes  3,324,108.00 bytes/sec
+total size is 1,657,278  speedup is 1.00

package/custom/composables/useAgentAudio.ts

@@ -14,118 +14,26 @@ type StreamingAudioState = {
   isDone: boolean;
 };
 
-let audioUnlockSourceUrl: string | null = null;
-let audioUnlockInFlight: Promise<void> | null = null;
-let isAudioPlaybackUnlocked = false;
 let standByAudio: HTMLAudioElement | null = null;
 let isStandByAudioPlaying = false;
-
-function writeAsciiString(view: DataView, offset: number, value: string) {
-  for (let index = 0; index < value.length; index += 1) {
-    view.setUint8(offset + index, value.charCodeAt(index));
-  }
-}
-
-function createSilentWavBlob(durationMs = 50) {
-  const sampleRate = 8000;
-  const bitsPerSample = 16;
-  const channelCount = 1;
-  const bytesPerSample = bitsPerSample / 8;
-  const sampleCount = Math.max(1, Math.round((sampleRate * durationMs) / 1000));
-  const pcmDataSize = sampleCount * channelCount * bytesPerSample;
-  const buffer = new ArrayBuffer(44 + pcmDataSize);
-  const view = new DataView(buffer);
-
-  writeAsciiString(view, 0, 'RIFF');
-  view.setUint32(4, 36 + pcmDataSize, true);
-  writeAsciiString(view, 8, 'WAVE');
-  writeAsciiString(view, 12, 'fmt ');
-  view.setUint32(16, 16, true);
-  view.setUint16(20, 1, true);
-  view.setUint16(22, channelCount, true);
-  view.setUint32(24, sampleRate, true);
-  view.setUint32(28, sampleRate * channelCount * bytesPerSample, true);
-  view.setUint16(32, channelCount * bytesPerSample, true);
-  view.setUint16(34, bitsPerSample, true);
-  writeAsciiString(view, 36, 'data');
-  view.setUint32(40, pcmDataSize, true);
-
-  return new Blob([buffer], { type: 'audio/wav' });
-}
-
-function getAudioUnlockSourceUrl() {
-  if (!audioUnlockSourceUrl) {
-    audioUnlockSourceUrl = URL.createObjectURL(createSilentWavBlob());
-  }
-
-  return audioUnlockSourceUrl;
-}
-
-async function unlockAudioPlayback() {
-  if (isAudioPlaybackUnlocked) {
-    return;
-  }
-
-  if (audioUnlockInFlight) {
-    await audioUnlockInFlight;
-    return;
-  }
-
-  audioUnlockInFlight = (async () => {
-    const unlockAudio = new Audio(getAudioUnlockSourceUrl());
-    unlockAudio.muted = true;
-    unlockAudio.preload = 'auto';
-    unlockAudio.setAttribute('playsinline', '');
-
-    try {
-      await unlockAudio.play();
-      unlockAudio.pause();
-      unlockAudio.currentTime = 0;
-      isAudioPlaybackUnlocked = true;
-    } catch (error) {
-      console.error('Failed to unlock audio playback:', error);
-    } finally {
-      unlockAudio.removeAttribute('src');
-      unlockAudio.load();
-      audioUnlockInFlight = null;
-    }
-  })();
-
-  await audioUnlockInFlight;
-}
-
 async function playStandByAudio() {
   if (!standByAudio) {
     standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
-    standByAudio.preload = 'auto';
-    standByAudio.setAttribute('playsinline', '');
     standByAudio.addEventListener('ended', () => {
-      if (standByAudio?.paused === false) {
+      if (!standByAudio.paused) {
         restartStandByAudio();
       }
     });
   }
-
-  if (!standByAudio) {
-    return;
-  }
-
   standByAudio.currentTime = 0;
-
-  try {
-    await standByAudio.play();
-    isStandByAudioPlaying = true;
-  } catch (error) {
-    isStandByAudioPlaying = false;
-    console.error('Failed to play standby audio:', error);
-  }
+  await standByAudio.play();
+  isStandByAudioPlaying = true;
 }
 
 function stopStandByAudio() {
   if (!standByAudio) {
     return;
   }
-
   standByAudio.pause();
   standByAudio.currentTime = 0;
   isStandByAudioPlaying = false;
@@ -135,15 +43,15 @@ function restartStandByAudio() {
   if (standByAudio) {
     standByAudio.currentTime = 0;
   }
-
-  void playStandByAudio();
+  playStandByAudio();
 }
 
+
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond'>('readyToRespond');
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
-
+    
   let currentAbortController: AbortController | null = null;
   let isPlaying = false;
   let currentAudio: HTMLAudioElement | null = null;
@@ -173,7 +81,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);
     formData.append('currentPage', JSON.stringify(getCurrentPageContext()));
     const fullPath = `${import.meta.env.VITE_ADMINFORTH_PUBLIC_PATH || ''}/adminapi/v1/agent/speech-response`;
-
     try {
       agentAudioMode.value = 'transcribing';
       const res = await fetch(fullPath, {
@@ -184,23 +91,22 @@ export const useAgentAudio = defineStore('agentAudio', () => {
         },
         signal: currentAbortController!.signal,
       });
-
       isStreamingResponse.value = true;
-
       if (res.ok) {
         agentAudioMode.value = 'streaming';
         await readSpeechResponseStream(res);
       } else {
         console.error('Failed to transcribe audio:', res.statusText);
-        adminforth.alert({ message: 'Failed to transcribe audio', variant: 'danger' });
+        adminforth.alert({message: 'Failed to transcribe audio', variant: 'danger'});
       }
     } catch (error) {
-      if (!(error instanceof DOMException && error.name === 'AbortError')) {
+      if (error instanceof DOMException && error.name === 'AbortError') {
+        //
+      } else {
         console.error('Error sending audio to server:', error);
       }
     } finally {
       isStreamingResponse.value = false;
-
       if (!wasAudioResponseReceived) {
         setAudioModeReadyToRespond();
       }
@@ -209,9 +115,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
   async function readSpeechResponseStream(res: Response) {
     const reader = res.body?.getReader();
-
     if (!reader) {
-      adminforth.alert({ message: 'Speech response stream is not available', variant: 'danger' });
+      adminforth.alert({message: 'Speech response stream is not available', variant: 'danger'});
       return;
     }
 
@@ -219,7 +124,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     let buffer = '';
 
     agentStore.setCurrentChatStatus('streaming');
-
     try {
       while (true) {
         const { value, done } = await reader.read();
@@ -254,7 +158,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     if (currentAbortController?.signal.aborted) {
       return;
     }
-
     const data = eventBlock
       .split('\n')
       .filter((line) => line.startsWith('data:'))
@@ -268,6 +171,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     const event = JSON.parse(data) as SpeechStreamEvent;
 
     if (event.type === 'error') {
+
       return;
     }
 
@@ -305,9 +209,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
     if (event.type === 'data-tool-call') {
       if (!isStandByAudioPlaying) {
-        void playStandByAudio();
+        playStandByAudio();
       }
-
       agentStore.addDataToolCallMessage(event.data);
     }
   }
@@ -324,16 +227,10 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       currentAudio.currentTime = 0;
       return;
     }
-
     agentAudioMode.value = 'playingAgentResponse';
-
-    try {
-      await currentAudio.play();
-    } catch (error) {
-      isPlaying = false;
-      setAudioModeReadyToRespond();
+    await void currentAudio.play().catch((error) => {
       console.error('Failed to play audio:', error);
-    }
+    });
   }
 
   function initializeAudioStream(mimeType: string) {
@@ -347,8 +244,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     const mediaSource = new MediaSource();
     currentAudioObjectUrl = URL.createObjectURL(mediaSource);
     currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.preload = 'auto';
-    currentAudio.setAttribute('playsinline', '');
     currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
     currentStreamingAudio = {
       mimeType,
@@ -404,7 +299,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
       if (!currentStreamingAudio.hasStartedPlayback) {
         currentStreamingAudio.hasStartedPlayback = true;
-        void setIsPlaying(true);
+        setIsPlaying(true);
       }
 
       return;
@@ -461,7 +356,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     bufferedAudioMimeType = 'audio/mpeg';
     detachStreamingAudio();
     destroyCurrentAudioElement();
-
     if (!dontResetMode) {
       setAudioModeReadyToRespond();
     }
@@ -475,10 +369,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
     currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
     currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.preload = 'auto';
-    currentAudio.setAttribute('playsinline', '');
     currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    void setIsPlaying(true);
+    setIsPlaying(true);
   }
 
   function base64ToArrayBuffer(base64: string) {
@@ -515,9 +407,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     sendAudioToServerAndHandleResponse,
     stopGenerationAndAudio,
     stopCurrentAudioPlayback,
-    unlockAudioPlayback,
     playBeep,
-    agentAudioMode,
-    playStandByAudio,
+    agentAudioMode
   };
+
 });

package/custom/conversation_area/TextRenderer.vue

@@ -1,9 +1,9 @@
 <template>
   <div
-    class="flex px-4 m-2 rounded-xl border border-gray-200 dark:border-gray-700"
+    class="flex rounded-xl border border-gray-200 dark:border-gray-700"
     @click="handleMarkdownLinkClick"
     :class="[
-      hasVegaLite ? 'w-full' : '',
+      hasVegaLite ? 'w-full px-6 my-2' : 'px-4 m-2',
       props.role === 'user' ? 'bg-lightListTableHeading dark:bg-darkListTableHeading self-end max-w-[80%] mr-4' 
         : 'border-none self-start'
     ]"

package/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -111,7 +111,6 @@ function toggleChatMode() {
 }
 
 async function onStartRecording() {
-  await agentAudio.unlockAudioPlayback();
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
   microphoneButtonMode.value = 'listen';
   agentAudio.playBeep(1000);

package/dist/custom/composables/useAgentAudio.ts

@@ -14,118 +14,26 @@ type StreamingAudioState = {
   isDone: boolean;
 };
 
-let audioUnlockSourceUrl: string | null = null;
-let audioUnlockInFlight: Promise<void> | null = null;
-let isAudioPlaybackUnlocked = false;
 let standByAudio: HTMLAudioElement | null = null;
 let isStandByAudioPlaying = false;
-
-function writeAsciiString(view: DataView, offset: number, value: string) {
-  for (let index = 0; index < value.length; index += 1) {
-    view.setUint8(offset + index, value.charCodeAt(index));
-  }
-}
-
-function createSilentWavBlob(durationMs = 50) {
-  const sampleRate = 8000;
-  const bitsPerSample = 16;
-  const channelCount = 1;
-  const bytesPerSample = bitsPerSample / 8;
-  const sampleCount = Math.max(1, Math.round((sampleRate * durationMs) / 1000));
-  const pcmDataSize = sampleCount * channelCount * bytesPerSample;
-  const buffer = new ArrayBuffer(44 + pcmDataSize);
-  const view = new DataView(buffer);
-
-  writeAsciiString(view, 0, 'RIFF');
-  view.setUint32(4, 36 + pcmDataSize, true);
-  writeAsciiString(view, 8, 'WAVE');
-  writeAsciiString(view, 12, 'fmt ');
-  view.setUint32(16, 16, true);
-  view.setUint16(20, 1, true);
-  view.setUint16(22, channelCount, true);
-  view.setUint32(24, sampleRate, true);
-  view.setUint32(28, sampleRate * channelCount * bytesPerSample, true);
-  view.setUint16(32, channelCount * bytesPerSample, true);
-  view.setUint16(34, bitsPerSample, true);
-  writeAsciiString(view, 36, 'data');
-  view.setUint32(40, pcmDataSize, true);
-
-  return new Blob([buffer], { type: 'audio/wav' });
-}
-
-function getAudioUnlockSourceUrl() {
-  if (!audioUnlockSourceUrl) {
-    audioUnlockSourceUrl = URL.createObjectURL(createSilentWavBlob());
-  }
-
-  return audioUnlockSourceUrl;
-}
-
-async function unlockAudioPlayback() {
-  if (isAudioPlaybackUnlocked) {
-    return;
-  }
-
-  if (audioUnlockInFlight) {
-    await audioUnlockInFlight;
-    return;
-  }
-
-  audioUnlockInFlight = (async () => {
-    const unlockAudio = new Audio(getAudioUnlockSourceUrl());
-    unlockAudio.muted = true;
-    unlockAudio.preload = 'auto';
-    unlockAudio.setAttribute('playsinline', '');
-
-    try {
-      await unlockAudio.play();
-      unlockAudio.pause();
-      unlockAudio.currentTime = 0;
-      isAudioPlaybackUnlocked = true;
-    } catch (error) {
-      console.error('Failed to unlock audio playback:', error);
-    } finally {
-      unlockAudio.removeAttribute('src');
-      unlockAudio.load();
-      audioUnlockInFlight = null;
-    }
-  })();
-
-  await audioUnlockInFlight;
-}
-
 async function playStandByAudio() {
   if (!standByAudio) {
     standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
-    standByAudio.preload = 'auto';
-    standByAudio.setAttribute('playsinline', '');
     standByAudio.addEventListener('ended', () => {
-      if (standByAudio?.paused === false) {
+      if (!standByAudio.paused) {
         restartStandByAudio();
       }
     });
   }
-
-  if (!standByAudio) {
-    return;
-  }
-
   standByAudio.currentTime = 0;
-
-  try {
-    await standByAudio.play();
-    isStandByAudioPlaying = true;
-  } catch (error) {
-    isStandByAudioPlaying = false;
-    console.error('Failed to play standby audio:', error);
-  }
+  await standByAudio.play();
+  isStandByAudioPlaying = true;
 }
 
 function stopStandByAudio() {
   if (!standByAudio) {
     return;
   }
-
   standByAudio.pause();
   standByAudio.currentTime = 0;
   isStandByAudioPlaying = false;
@@ -135,15 +43,15 @@ function restartStandByAudio() {
   if (standByAudio) {
     standByAudio.currentTime = 0;
   }
-
-  void playStandByAudio();
+  playStandByAudio();
 }
 
+
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond'>('readyToRespond');
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
-
+    
   let currentAbortController: AbortController | null = null;
   let isPlaying = false;
   let currentAudio: HTMLAudioElement | null = null;
@@ -173,7 +81,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);
     formData.append('currentPage', JSON.stringify(getCurrentPageContext()));
     const fullPath = `${import.meta.env.VITE_ADMINFORTH_PUBLIC_PATH || ''}/adminapi/v1/agent/speech-response`;
-
     try {
       agentAudioMode.value = 'transcribing';
       const res = await fetch(fullPath, {
@@ -184,23 +91,22 @@ export const useAgentAudio = defineStore('agentAudio', () => {
         },
         signal: currentAbortController!.signal,
       });
-
       isStreamingResponse.value = true;
-
       if (res.ok) {
         agentAudioMode.value = 'streaming';
         await readSpeechResponseStream(res);
       } else {
         console.error('Failed to transcribe audio:', res.statusText);
-        adminforth.alert({ message: 'Failed to transcribe audio', variant: 'danger' });
+        adminforth.alert({message: 'Failed to transcribe audio', variant: 'danger'});
       }
     } catch (error) {
-      if (!(error instanceof DOMException && error.name === 'AbortError')) {
+      if (error instanceof DOMException && error.name === 'AbortError') {
+        //
+      } else {
         console.error('Error sending audio to server:', error);
       }
     } finally {
       isStreamingResponse.value = false;
-
       if (!wasAudioResponseReceived) {
         setAudioModeReadyToRespond();
       }
@@ -209,9 +115,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
   async function readSpeechResponseStream(res: Response) {
     const reader = res.body?.getReader();
-
     if (!reader) {
-      adminforth.alert({ message: 'Speech response stream is not available', variant: 'danger' });
+      adminforth.alert({message: 'Speech response stream is not available', variant: 'danger'});
       return;
     }
 
@@ -219,7 +124,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     let buffer = '';
 
     agentStore.setCurrentChatStatus('streaming');
-
     try {
       while (true) {
         const { value, done } = await reader.read();
@@ -254,7 +158,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     if (currentAbortController?.signal.aborted) {
       return;
     }
-
     const data = eventBlock
       .split('\n')
       .filter((line) => line.startsWith('data:'))
@@ -268,6 +171,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     const event = JSON.parse(data) as SpeechStreamEvent;
 
     if (event.type === 'error') {
+
       return;
     }
 
@@ -305,9 +209,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
     if (event.type === 'data-tool-call') {
       if (!isStandByAudioPlaying) {
-        void playStandByAudio();
+        playStandByAudio();
       }
-
       agentStore.addDataToolCallMessage(event.data);
     }
   }
@@ -324,16 +227,10 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       currentAudio.currentTime = 0;
       return;
     }
-
     agentAudioMode.value = 'playingAgentResponse';
-
-    try {
-      await currentAudio.play();
-    } catch (error) {
-      isPlaying = false;
-      setAudioModeReadyToRespond();
+    await void currentAudio.play().catch((error) => {
       console.error('Failed to play audio:', error);
-    }
+    });
   }
 
   function initializeAudioStream(mimeType: string) {
@@ -347,8 +244,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     const mediaSource = new MediaSource();
     currentAudioObjectUrl = URL.createObjectURL(mediaSource);
     currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.preload = 'auto';
-    currentAudio.setAttribute('playsinline', '');
     currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
     currentStreamingAudio = {
       mimeType,
@@ -404,7 +299,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
       if (!currentStreamingAudio.hasStartedPlayback) {
         currentStreamingAudio.hasStartedPlayback = true;
-        void setIsPlaying(true);
+        setIsPlaying(true);
       }
 
       return;
@@ -461,7 +356,6 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     bufferedAudioMimeType = 'audio/mpeg';
     detachStreamingAudio();
     destroyCurrentAudioElement();
-
     if (!dontResetMode) {
       setAudioModeReadyToRespond();
     }
@@ -475,10 +369,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
     currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
     currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.preload = 'auto';
-    currentAudio.setAttribute('playsinline', '');
     currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    void setIsPlaying(true);
+    setIsPlaying(true);
   }
 
   function base64ToArrayBuffer(base64: string) {
@@ -515,9 +407,8 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     sendAudioToServerAndHandleResponse,
     stopGenerationAndAudio,
     stopCurrentAudioPlayback,
-    unlockAudioPlayback,
     playBeep,
-    agentAudioMode,
-    playStandByAudio,
+    agentAudioMode
   };
+
 });

package/dist/custom/conversation_area/TextRenderer.vue

@@ -1,9 +1,9 @@
 <template>
   <div
-    class="flex px-4 m-2 rounded-xl border border-gray-200 dark:border-gray-700"
+    class="flex rounded-xl border border-gray-200 dark:border-gray-700"
     @click="handleMarkdownLinkClick"
     :class="[
-      hasVegaLite ? 'w-full' : '',
+      hasVegaLite ? 'w-full px-6 my-2' : 'px-4 m-2',
       props.role === 'user' ? 'bg-lightListTableHeading dark:bg-darkListTableHeading self-end max-w-[80%] mr-4' 
         : 'border-none self-start'
     ]"

package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -111,7 +111,6 @@ function toggleChatMode() {
 }
 
 async function onStartRecording() {
-  await agentAudio.unlockAudioPlayback();
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
   microphoneButtonMode.value = 'listen';
   agentAudio.playBeep(1000);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.43.3",
+  "version": "1.43.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",