@adminforth/agent 1.41.1 → 1.41.3

package/build.log

@@ -58,5 +58,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
 custom/speech_recognition_frontend/types/
 custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
 
-sent 1,660,143 bytes  received 856 bytes  3,321,998.00 bytes/sec
-total size is 1,656,231  speedup is 1.00
+sent 1,660,390 bytes  received 860 bytes  3,322,500.00 bytes/sec
+total size is 1,656,528  speedup is 1.00

package/custom/composables/useAgentAudio.ts

@@ -15,7 +15,7 @@ type StreamingAudioState = {
 };
 
 let standByAudio: HTMLAudioElement | null = null;
-
+let isStandByAudioPlaying = false;
 function playStandByAudio() {
   if (!standByAudio) {
     standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
@@ -27,6 +27,7 @@ function playStandByAudio() {
   }
   standByAudio.currentTime = 0;
   standByAudio.play()
+  isStandByAudioPlaying = true;
 }
 
 function stopStandByAudio() {
@@ -35,6 +36,7 @@ function stopStandByAudio() {
   }
   standByAudio.pause();
   standByAudio.currentTime = 0;
+  isStandByAudioPlaying = false;
 }
 
 function restartStandByAudio() {
@@ -47,7 +49,7 @@ function restartStandByAudio() {
 
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
     
   let currentAbortController: AbortController | null = null;
@@ -57,15 +59,21 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   let currentStreamingAudio: StreamingAudioState | null = null;
   let bufferedAudioChunks: ArrayBuffer[] = [];
   let bufferedAudioMimeType = 'audio/mpeg';
+  let wasAudioResponseReceived = false;
 
   function stopGenerationAndAudio() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
     currentAbortController?.abort();
   }
 
+  function setAudioModeReadyToRespond() {
+    agentAudioMode.value = 'readyToRespond';
+  }
+
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
     currentAbortController = new AbortController();
+    wasAudioResponseReceived = false;
     const formData = new FormData();
     formData.append('file', blob, 'user_prompt.wav');
     formData.append('sessionId', agentStore.activeSessionId);
@@ -99,7 +107,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       }
     } finally {
       isStreamingResponse.value = false;
-      agentAudioMode.value = null;
+      if (!wasAudioResponseReceived) {
+        setAudioModeReadyToRespond();
+      }
     }
   }
 
@@ -175,14 +185,15 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       stopStandByAudio();
       agentStore.setCurrentChatStatus('ready');
       agentStore.addAgentMessage(event.data.response.text);
-      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
     if (event.type === 'audio-start') {
+      wasAudioResponseReceived = true;
       isStreamingResponse.value = false;
       agentAudioMode.value = 'fetchingAudio';
       initializeAudioStream(event.data.mimeType);
+      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
@@ -197,7 +208,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
 
     if (event.type === 'data-tool-call') {
-      playStandByAudio();
+      if (!isStandByAudioPlaying) {
+        playStandByAudio();
+      }
       agentStore.addDataToolCallMessage(event.data);
     }
   }
@@ -344,12 +357,12 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     detachStreamingAudio();
     destroyCurrentAudioElement();
     if (!dontResetMode) {
-      agentAudioMode.value = null;
+      setAudioModeReadyToRespond();
     }
   }
 
   function handleAudioEnded() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
   }
 

package/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -63,7 +63,7 @@ onMounted(() => {
   });
 });
 
-watch(agentAudioMode, (newVal) => {
+watch(agentAudioMode, async (newVal) => {
   if(newVal === 'streaming') {
     stopCurrentAudioPlayback(true);
     microphoneButtonMode.value = 'generating';
@@ -72,10 +72,11 @@ watch(agentAudioMode, (newVal) => {
   } else if (newVal === 'fetchingAudio') {
     //Generation is done, waiting for audio to be ready
   } else if (newVal === 'playingAgentResponse') {
-    // Audio is playing
-  } else {
+    // Audio is playing' 
+  } else if (newVal === 'readyToRespond') {
     if(isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
+      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
     } else {
       microphoneButtonMode.value = 'off';
     }
@@ -94,8 +95,8 @@ function toggleChatMode() {
 
 async function onStartRecording() {
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-      microphoneButtonMode.value = 'listen';
-      agentAudio.playBeep(1000);
+  microphoneButtonMode.value = 'listen';
+  agentAudio.playBeep(1000);
 }
 
 function onStopRecording() {
@@ -142,12 +143,8 @@ async function sendRecordForTranscription() {
   showAudioWavesAnimation.value = false;
   const recordBlob = await getRecord();
   if (recordBlob) {
-    console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
     onStopRecording();
     await sendAudioToServerAndHandleResponse(recordBlob);
-    if (agentStore.isAudioChatMode) {
-      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-    }
   } else { 
     console.error('No audio recorded');
   }

package/dist/custom/composables/useAgentAudio.ts

@@ -15,7 +15,7 @@ type StreamingAudioState = {
 };
 
 let standByAudio: HTMLAudioElement | null = null;
-
+let isStandByAudioPlaying = false;
 function playStandByAudio() {
   if (!standByAudio) {
     standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
@@ -27,6 +27,7 @@ function playStandByAudio() {
   }
   standByAudio.currentTime = 0;
   standByAudio.play()
+  isStandByAudioPlaying = true;
 }
 
 function stopStandByAudio() {
@@ -35,6 +36,7 @@ function stopStandByAudio() {
   }
   standByAudio.pause();
   standByAudio.currentTime = 0;
+  isStandByAudioPlaying = false;
 }
 
 function restartStandByAudio() {
@@ -47,7 +49,7 @@ function restartStandByAudio() {
 
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
     
   let currentAbortController: AbortController | null = null;
@@ -57,15 +59,21 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   let currentStreamingAudio: StreamingAudioState | null = null;
   let bufferedAudioChunks: ArrayBuffer[] = [];
   let bufferedAudioMimeType = 'audio/mpeg';
+  let wasAudioResponseReceived = false;
 
   function stopGenerationAndAudio() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
     currentAbortController?.abort();
   }
 
+  function setAudioModeReadyToRespond() {
+    agentAudioMode.value = 'readyToRespond';
+  }
+
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
     currentAbortController = new AbortController();
+    wasAudioResponseReceived = false;
     const formData = new FormData();
     formData.append('file', blob, 'user_prompt.wav');
     formData.append('sessionId', agentStore.activeSessionId);
@@ -99,7 +107,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       }
     } finally {
       isStreamingResponse.value = false;
-      agentAudioMode.value = null;
+      if (!wasAudioResponseReceived) {
+        setAudioModeReadyToRespond();
+      }
     }
   }
 
@@ -175,14 +185,15 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       stopStandByAudio();
       agentStore.setCurrentChatStatus('ready');
       agentStore.addAgentMessage(event.data.response.text);
-      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
     if (event.type === 'audio-start') {
+      wasAudioResponseReceived = true;
       isStreamingResponse.value = false;
       agentAudioMode.value = 'fetchingAudio';
       initializeAudioStream(event.data.mimeType);
+      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
@@ -197,7 +208,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
 
     if (event.type === 'data-tool-call') {
-      playStandByAudio();
+      if (!isStandByAudioPlaying) {
+        playStandByAudio();
+      }
       agentStore.addDataToolCallMessage(event.data);
     }
   }
@@ -344,12 +357,12 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     detachStreamingAudio();
     destroyCurrentAudioElement();
     if (!dontResetMode) {
-      agentAudioMode.value = null;
+      setAudioModeReadyToRespond();
     }
   }
 
   function handleAudioEnded() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
   }
 

package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -63,7 +63,7 @@ onMounted(() => {
   });
 });
 
-watch(agentAudioMode, (newVal) => {
+watch(agentAudioMode, async (newVal) => {
   if(newVal === 'streaming') {
     stopCurrentAudioPlayback(true);
     microphoneButtonMode.value = 'generating';
@@ -72,10 +72,11 @@ watch(agentAudioMode, (newVal) => {
   } else if (newVal === 'fetchingAudio') {
     //Generation is done, waiting for audio to be ready
   } else if (newVal === 'playingAgentResponse') {
-    // Audio is playing
-  } else {
+    // Audio is playing' 
+  } else if (newVal === 'readyToRespond') {
     if(isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
+      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
     } else {
       microphoneButtonMode.value = 'off';
     }
@@ -94,8 +95,8 @@ function toggleChatMode() {
 
 async function onStartRecording() {
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-      microphoneButtonMode.value = 'listen';
-      agentAudio.playBeep(1000);
+  microphoneButtonMode.value = 'listen';
+  agentAudio.playBeep(1000);
 }
 
 function onStopRecording() {
@@ -142,12 +143,8 @@ async function sendRecordForTranscription() {
   showAudioWavesAnimation.value = false;
   const recordBlob = await getRecord();
   if (recordBlob) {
-    console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
     onStopRecording();
     await sendAudioToServerAndHandleResponse(recordBlob);
-    if (agentStore.isAudioChatMode) {
-      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-    }
   } else { 
     console.error('No audio recorded');
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.41.1",
+  "version": "1.41.3",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",