@adminforth/agent 1.40.4 → 1.41.1

package/build.log

@@ -58,5 +58,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
 custom/speech_recognition_frontend/types/
 custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
 
-sent 1,657,857 bytes  received 860 bytes  3,317,434.00 bytes/sec
-total size is 1,653,941  speedup is 1.00
+sent 1,660,143 bytes  received 856 bytes  3,321,998.00 bytes/sec
+total size is 1,656,231  speedup is 1.00

package/custom/composables/useAgentAudio.ts

@@ -67,7 +67,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
     currentAbortController = new AbortController();
     const formData = new FormData();
-    formData.append('file', blob, 'user_prompt.webm');
+    formData.append('file', blob, 'user_prompt.wav');
     formData.append('sessionId', agentStore.activeSessionId);
     formData.append('mode', agentStore.activeModeName ?? '');
     formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);

package/custom/package.json

@@ -14,6 +14,7 @@
     "@incremark/core": "^1.0.2",
     "@incremark/theme": "^1.0.2",
     "@incremark/vue": "^1.0.2",
+    "@ricky0123/vad-web": "^0.0.30",
     "@shikijs/langs": "^4.0.2",
     "@shikijs/themes": "^4.0.2",
     "@vueuse/core": "^14.2.1",

package/custom/pnpm-lock.yaml

@@ -17,6 +17,9 @@ importers:
       '@incremark/vue':
         specifier: ^1.0.2
         version: 1.0.2(katex@0.16.45)(vue@3.5.32)
+      '@ricky0123/vad-web':
+        specifier: ^0.0.30
+        version: 0.0.30
       '@shikijs/langs':
         specifier: ^4.0.2
         version: 4.0.2
@@ -139,6 +142,39 @@ packages:
     resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
     engines: {node: '>=8.0.0'}
 
+  '@protobufjs/aspromise@1.1.2':
+    resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
+
+  '@protobufjs/base64@1.1.2':
+    resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
+
+  '@protobufjs/codegen@2.0.5':
+    resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==}
+
+  '@protobufjs/eventemitter@1.1.0':
+    resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
+
+  '@protobufjs/fetch@1.1.0':
+    resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
+
+  '@protobufjs/float@1.0.2':
+    resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
+
+  '@protobufjs/inquire@1.1.1':
+    resolution: {integrity: sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==}
+
+  '@protobufjs/path@1.1.2':
+    resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
+
+  '@protobufjs/pool@1.1.0':
+    resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
+
+  '@protobufjs/utf8@1.1.1':
+    resolution: {integrity: sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==}
+
+  '@ricky0123/vad-web@0.0.30':
+    resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==}
+
   '@shikijs/core@3.23.0':
     resolution: {integrity: sha512-NSWQz0riNb67xthdm5br6lAkvpDJRTgB36fxlo37ZzM2yq0PQFFzbd8psqC2XMPgCzo1fW6cVi18+ArJ44wqgA==}
 
@@ -218,6 +254,9 @@ packages:
   '@types/ms@2.1.0':
     resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
 
+  '@types/node@25.6.0':
+    resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}
+
   '@types/trusted-types@2.0.7':
     resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==}
 
@@ -472,6 +511,9 @@ packages:
   fast-json-patch@3.1.1:
     resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==}
 
+  flatbuffers@25.9.23:
+    resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
+
   get-caller-file@2.0.5:
     resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
     engines: {node: 6.* || 8.* || >= 10.*}
@@ -480,6 +522,9 @@ packages:
     resolution: {integrity: sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==}
     engines: {node: '>=18'}
 
+  guid-typescript@1.0.9:
+    resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
+
   hast-util-to-html@9.0.5:
     resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==}
 
@@ -525,6 +570,9 @@ packages:
   lodash-es@4.18.1:
     resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==}
 
+  long@5.3.2:
+    resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
+
   longest-streak@3.1.0:
     resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==}
 
@@ -690,12 +738,21 @@ packages:
   oniguruma-to-es@4.3.5:
     resolution: {integrity: sha512-Zjygswjpsewa0NLTsiizVuMQZbp0MDyM6lIt66OxsF21npUDlzpHi1Mgb/qhQdkb+dWFTzJmFbEWdvZgRho8eQ==}
 
+  onnxruntime-common@1.25.1:
+    resolution: {integrity: sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg==}
+
+  onnxruntime-web@1.25.1:
+    resolution: {integrity: sha512-mgs61sJ9m3hLa5jGRr9Pen3kkG00vlxmrcRL6FufYpSWBZKaklo0sotQCq2fLjgDVLnW57jrDcLqzYJNKeZskQ==}
+
   parse-entities@4.0.2:
     resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
 
   picocolors@1.1.1:
     resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
 
+  platform@1.3.6:
+    resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
+
   postcss@8.5.10:
     resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==}
     engines: {node: ^10 || ^12 || >=14}
@@ -703,6 +760,10 @@ packages:
   property-information@7.1.0:
     resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==}
 
+  protobufjs@7.5.6:
+    resolution: {integrity: sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg==}
+    engines: {node: '>=12.0.0'}
+
   regex-recursion@6.0.2:
     resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}
 
@@ -775,6 +836,9 @@ packages:
   tslib@2.8.1:
     resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==}
 
+  undici-types@7.19.2:
+    resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==}
+
   unist-util-is@6.0.1:
     resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==}
 
@@ -1063,6 +1127,33 @@ snapshots:
 
   '@opentelemetry/api@1.9.0': {}
 
+  '@protobufjs/aspromise@1.1.2': {}
+
+  '@protobufjs/base64@1.1.2': {}
+
+  '@protobufjs/codegen@2.0.5': {}
+
+  '@protobufjs/eventemitter@1.1.0': {}
+
+  '@protobufjs/fetch@1.1.0':
+    dependencies:
+      '@protobufjs/aspromise': 1.1.2
+      '@protobufjs/inquire': 1.1.1
+
+  '@protobufjs/float@1.0.2': {}
+
+  '@protobufjs/inquire@1.1.1': {}
+
+  '@protobufjs/path@1.1.2': {}
+
+  '@protobufjs/pool@1.1.0': {}
+
+  '@protobufjs/utf8@1.1.1': {}
+
+  '@ricky0123/vad-web@0.0.30':
+    dependencies:
+      onnxruntime-web: 1.25.1
+
   '@shikijs/core@3.23.0':
     dependencies:
       '@shikijs/types': 3.23.0
@@ -1162,6 +1253,10 @@ snapshots:
 
   '@types/ms@2.1.0': {}
 
+  '@types/node@25.6.0':
+    dependencies:
+      undici-types: 7.19.2
+
   '@types/trusted-types@2.0.7':
     optional: true
 
@@ -1404,10 +1499,14 @@ snapshots:
 
   fast-json-patch@3.1.1: {}
 
+  flatbuffers@25.9.23: {}
+
   get-caller-file@2.0.5: {}
 
   get-east-asian-width@1.5.0: {}
 
+  guid-typescript@1.0.9: {}
+
   hast-util-to-html@9.0.5:
     dependencies:
       '@types/hast': 3.0.4
@@ -1457,6 +1556,8 @@ snapshots:
 
   lodash-es@4.18.1: {}
 
+  long@5.3.2: {}
+
   longest-streak@3.1.0: {}
 
   magic-string@0.30.21:
@@ -1832,6 +1933,17 @@ snapshots:
       regex: 6.1.0
       regex-recursion: 6.0.2
 
+  onnxruntime-common@1.25.1: {}
+
+  onnxruntime-web@1.25.1:
+    dependencies:
+      flatbuffers: 25.9.23
+      guid-typescript: 1.0.9
+      long: 5.3.2
+      onnxruntime-common: 1.25.1
+      platform: 1.3.6
+      protobufjs: 7.5.6
+
   parse-entities@4.0.2:
     dependencies:
       '@types/unist': 2.0.11
@@ -1844,6 +1956,8 @@ snapshots:
 
   picocolors@1.1.1: {}
 
+  platform@1.3.6: {}
+
   postcss@8.5.10:
     dependencies:
       nanoid: 3.3.11
@@ -1852,6 +1966,21 @@ snapshots:
 
   property-information@7.1.0: {}
 
+  protobufjs@7.5.6:
+    dependencies:
+      '@protobufjs/aspromise': 1.1.2
+      '@protobufjs/base64': 1.1.2
+      '@protobufjs/codegen': 2.0.5
+      '@protobufjs/eventemitter': 1.1.0
+      '@protobufjs/fetch': 1.1.0
+      '@protobufjs/float': 1.0.2
+      '@protobufjs/inquire': 1.1.1
+      '@protobufjs/path': 1.1.2
+      '@protobufjs/pool': 1.1.0
+      '@protobufjs/utf8': 1.1.1
+      '@types/node': 25.6.0
+      long: 5.3.2
+
   regex-recursion@6.0.2:
     dependencies:
       regex-utilities: 2.3.0
@@ -1925,6 +2054,8 @@ snapshots:
 
   tslib@2.8.1: {}
 
+  undici-types@7.19.2: {}
+
   unist-util-is@6.0.1:
     dependencies:
       '@types/unist': 3.0.3

package/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -28,7 +28,7 @@
 <script setup lang="ts">
 import { computed, onMounted, onBeforeUnmount, ref, watch } from 'vue';
 import debounce from 'lodash.debounce';
-import { requestMicAndStartVAD, stopUserMedia, getRecorder, CALIBRATION_DURATION } from './voiceActivityDetection';
+import { requestMicAndStartVAD, stopUserMedia, getRecord } from './voiceActivityDetection';
 import { Spinner } from '@/afcl'
 import { storeToRefs } from 'pinia';
 import { useAgentStore } from '../composables/useAgentStore';
@@ -41,7 +41,7 @@ const { sendAudioToServerAndHandleResponse } = agentAudio;
 const { stopGenerationAndAudio } = agentAudio;
 const { stopCurrentAudioPlayback } = agentAudio;
 const { agentAudioMode } = storeToRefs(agentAudio);
-const microphoneButtonMode = ref<'off' | 'calibrating' | 'listen' | 'transcribing' | 'generating'>('off');
+const microphoneButtonMode = ref<'off' | 'listen' | 'transcribing' | 'generating'>('off');
 const showAudioWavesAnimation = ref(false);
 const audioAmplitude = ref(0);
 const hideAnimationDebounced = debounce(() => {
@@ -93,14 +93,9 @@ function toggleChatMode() {
 }
 
 async function onStartRecording() {
-  microphoneButtonMode.value = 'calibrating';
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-  setTimeout(() => {
-    if (isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
       agentAudio.playBeep(1000);
-    }
-  }, CALIBRATION_DURATION);
 }
 
 function onStopRecording() {
@@ -145,8 +140,9 @@ function onAnySound(amplitude: number) {
 
 async function sendRecordForTranscription() {
   showAudioWavesAnimation.value = false;
-  const recordBlob = await getRecorder();
+  const recordBlob = await getRecord();
   if (recordBlob) {
+    console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
     onStopRecording();
     await sendAudioToServerAndHandleResponse(recordBlob);
     if (agentStore.isAudioChatMode) {

package/custom/speech_recognition_frontend/voiceActivityDetection.ts

@@ -1,151 +1,62 @@
-import vad from 'voice-activity-detection';
-
-let currentStream: MediaStream | null = null;
-let vadInstance: any = null;
-let audioContext: AudioContext | null = null;
-let mediaRecorder: MediaRecorder | null = null;
-let recordedChunks: BlobPart[] = [];
-let wasVoiceStarted = false;
-
-export const CALIBRATION_DURATION = 1000; // in ms
+import { MicVAD, utils } from "@ricky0123/vad-web"
+
+let VADInstance: MicVAD | null = null;
+let recordedAudioChunks: Float32Array[] = [];
+let onVoiceStopCallback: () => void = () => {};
+let onVoiceStartCallback: () => void = () => {};
+let onUpdateCallback: (amplitude: number) => void = () => {};
+
+async function createVADInstance(){
+  VADInstance = await MicVAD.new({
+    onFrameProcessed: ({ isSpeech }) => {
+      onUpdateCallback(isSpeech);
+    },
+    onSpeechEnd: (audio) => {
+      recordedAudioChunks.push(audio);
+      onVoiceStopCallback();
+    },
+    onSpeechStart: () => {
+      onVoiceStartCallback();
+    },
+    onnxWASMBasePath: "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/",
+    baseAssetPath: "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/",
+  })
+}
 
 export async function requestMicAndStartVAD(
-  onVoiceStopCallback: () => void,
-  onVoiceStartCallback: () => void,
-  onUpdateCallback: (amplitude: number) => void
+  onVoiceStop: () => void,
+  onVoiceStart: () => void,
+  onUpdate: (amplitude: number) => void
 ) {
-  return new Promise<void>((resolve, reject) => {
-    try {
-      audioContext = new AudioContext();
-
-      navigator.mediaDevices
-        .getUserMedia({ audio: true })
-        .then((stream) => {
-          currentStream = stream;
-          startRecording(stream);
-          resolve();
-          startUserMedia(audioContext as AudioContext, stream, onVoiceStartCallback, onVoiceStopCallback, onUpdateCallback);
-        })
-        .catch((error) => {
-          handleMicConnectError();
-          reject(error);
-        });
-    } catch (e) {
-      handleUserMediaError();
-      reject(e);
-    }
-  });
-}
-
-function handleUserMediaError() {
-  console.error('Mic input is not supported by the browser.');
-}
-
-function handleMicConnectError() {
-  console.error('Could not connect microphone. Possible rejected by the user or is blocked by the browser.');
-}
-
-export async function stopUserMedia() {
-  wasVoiceStarted = false;
-  if (vadInstance && vadInstance.destroy) {
-    vadInstance.destroy();
-    vadInstance = null;
-  }
-
-  if (currentStream) {
-    currentStream.getTracks().forEach(track => track.stop());
-    currentStream = null;
-  }
-
-  if (audioContext) {
-    audioContext.close();
-    audioContext = null;
-  }
+  onVoiceStopCallback = onVoiceStop;
+  onVoiceStartCallback = onVoiceStart;
+  onUpdateCallback = onUpdate;
 
-  if (mediaRecorder) {
-    mediaRecorder.stop();
-    mediaRecorder = null;
+  if (!VADInstance) {
+    await createVADInstance();
   }
-
+  VADInstance?.start(); 
 }
 
-function startRecording(stream: MediaStream) {
-  recordedChunks = [];
-  mediaRecorder = new MediaRecorder(stream);
-  mediaRecorder.ondataavailable = (event: BlobEvent) => {
-    if (event.data.size > 0) {
-      recordedChunks.push(event.data);
-    }
-  };
-  mediaRecorder.start();
+export async function stopUserMedia() {
+  await VADInstance?.pause();
 }
 
-export async function getRecorder(): Promise<Blob | null> {
-  if (!mediaRecorder) {
-    return Promise.resolve(null);
+export async function getRecord() {
+  const totalSamples = recordedAudioChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  if (totalSamples === 0) {
+    return null;
   }
 
-  const recorder = mediaRecorder;
-  mediaRecorder = null;
-
-  const finalizeBlob = () => {
-    const blob = new Blob(recordedChunks, { type: recorder.mimeType || 'audio/webm' });
-    recordedChunks = [];
-    return blob;
-  };
-
-  if (recorder.state === 'inactive') {
-    return Promise.resolve(finalizeBlob());
+  const mergedAudio = new Float32Array(totalSamples);
+  let offset = 0;
+  for (const chunk of recordedAudioChunks) {
+    mergedAudio.set(chunk, offset);
+    offset += chunk.length;
   }
 
-  return new Promise<Blob>((resolve, reject) => {
-    recorder.onstop = () => {
-      resolve(finalizeBlob());
-    };
-    recorder.onerror = () => {
-      recordedChunks = [];
-      reject(new Error('Failed to finalize audio recording.'));
-    };
-    recorder.stop();
-  });
-}
-
-function startUserMedia(
-  audioContext: AudioContext, 
-  stream: MediaStream, 
-  onVoiceStartCallback: () => void,
-  onVoiceStopCallback: () => void,
-  onUpdateCallback: (amplitude: number) => void
-) {
-  const options = {
-    fftSize: 1024,
-    bufferLen: 1024,
-    smoothingTimeConstant: 0.2,
-    minCaptureFreq: 85,         // in Hz
-    maxCaptureFreq: 255,        // in Hz
-    noiseCaptureDuration: CALIBRATION_DURATION, // in ms
-    minNoiseLevel: 0.5,         // from 0 to 1
-    maxNoiseLevel: 0.7,         // from 0 to 1
-    avgNoiseMultiplier: 1.2,
-    onVoiceStart() {
-      wasVoiceStarted = true;
-      if (!mediaRecorder || mediaRecorder.state === 'inactive') {
-        startRecording(currentStream as MediaStream);
-      }
-      console.log('👹👹👹voice start👹👹👹');
-      onVoiceStartCallback();
-    },
-    onVoiceStop() {
-      if (!wasVoiceStarted) {
-        return;
-      }
-      console.log('👿👿👿voice stop👿👿👿');
-      onVoiceStopCallback();
-    }, //Doesn't work properly, so we will handle it with onUpdate callback
-    onUpdate(val: number) {
-      onUpdateCallback(val);
-    }
-  };
-
-  vadInstance = vad(audioContext, stream, options);
+  const wavBuffer = utils.encodeWAV(mergedAudio, 1, 16000, 1, 16);
+  const recordToReturn = new Blob([wavBuffer], { type: 'audio/wav' });
+  recordedAudioChunks = [];
+  return recordToReturn;
 }

package/dist/custom/composables/useAgentAudio.ts

@@ -67,7 +67,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
     currentAbortController = new AbortController();
     const formData = new FormData();
-    formData.append('file', blob, 'user_prompt.webm');
+    formData.append('file', blob, 'user_prompt.wav');
     formData.append('sessionId', agentStore.activeSessionId);
     formData.append('mode', agentStore.activeModeName ?? '');
     formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);

package/dist/custom/package.json

@@ -14,6 +14,7 @@
     "@incremark/core": "^1.0.2",
     "@incremark/theme": "^1.0.2",
     "@incremark/vue": "^1.0.2",
+    "@ricky0123/vad-web": "^0.0.30",
     "@shikijs/langs": "^4.0.2",
     "@shikijs/themes": "^4.0.2",
     "@vueuse/core": "^14.2.1",

package/dist/custom/pnpm-lock.yaml

@@ -17,6 +17,9 @@ importers:
       '@incremark/vue':
         specifier: ^1.0.2
         version: 1.0.2(katex@0.16.45)(vue@3.5.32)
+      '@ricky0123/vad-web':
+        specifier: ^0.0.30
+        version: 0.0.30
       '@shikijs/langs':
         specifier: ^4.0.2
         version: 4.0.2
@@ -139,6 +142,39 @@ packages:
     resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
     engines: {node: '>=8.0.0'}
 
+  '@protobufjs/aspromise@1.1.2':
+    resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
+
+  '@protobufjs/base64@1.1.2':
+    resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
+
+  '@protobufjs/codegen@2.0.5':
+    resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==}
+
+  '@protobufjs/eventemitter@1.1.0':
+    resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
+
+  '@protobufjs/fetch@1.1.0':
+    resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
+
+  '@protobufjs/float@1.0.2':
+    resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
+
+  '@protobufjs/inquire@1.1.1':
+    resolution: {integrity: sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==}
+
+  '@protobufjs/path@1.1.2':
+    resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
+
+  '@protobufjs/pool@1.1.0':
+    resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
+
+  '@protobufjs/utf8@1.1.1':
+    resolution: {integrity: sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==}
+
+  '@ricky0123/vad-web@0.0.30':
+    resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==}
+
   '@shikijs/core@3.23.0':
     resolution: {integrity: sha512-NSWQz0riNb67xthdm5br6lAkvpDJRTgB36fxlo37ZzM2yq0PQFFzbd8psqC2XMPgCzo1fW6cVi18+ArJ44wqgA==}
 
@@ -218,6 +254,9 @@ packages:
   '@types/ms@2.1.0':
     resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
 
+  '@types/node@25.6.0':
+    resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}
+
   '@types/trusted-types@2.0.7':
     resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==}
 
@@ -472,6 +511,9 @@ packages:
   fast-json-patch@3.1.1:
     resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==}
 
+  flatbuffers@25.9.23:
+    resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
+
   get-caller-file@2.0.5:
     resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
     engines: {node: 6.* || 8.* || >= 10.*}
@@ -480,6 +522,9 @@ packages:
     resolution: {integrity: sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==}
     engines: {node: '>=18'}
 
+  guid-typescript@1.0.9:
+    resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
+
   hast-util-to-html@9.0.5:
     resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==}
 
@@ -525,6 +570,9 @@ packages:
   lodash-es@4.18.1:
     resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==}
 
+  long@5.3.2:
+    resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
+
   longest-streak@3.1.0:
     resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==}
 
@@ -690,12 +738,21 @@ packages:
   oniguruma-to-es@4.3.5:
     resolution: {integrity: sha512-Zjygswjpsewa0NLTsiizVuMQZbp0MDyM6lIt66OxsF21npUDlzpHi1Mgb/qhQdkb+dWFTzJmFbEWdvZgRho8eQ==}
 
+  onnxruntime-common@1.25.1:
+    resolution: {integrity: sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg==}
+
+  onnxruntime-web@1.25.1:
+    resolution: {integrity: sha512-mgs61sJ9m3hLa5jGRr9Pen3kkG00vlxmrcRL6FufYpSWBZKaklo0sotQCq2fLjgDVLnW57jrDcLqzYJNKeZskQ==}
+
   parse-entities@4.0.2:
     resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
 
   picocolors@1.1.1:
     resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
 
+  platform@1.3.6:
+    resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
+
   postcss@8.5.10:
     resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==}
     engines: {node: ^10 || ^12 || >=14}
@@ -703,6 +760,10 @@ packages:
   property-information@7.1.0:
     resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==}
 
+  protobufjs@7.5.6:
+    resolution: {integrity: sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg==}
+    engines: {node: '>=12.0.0'}
+
   regex-recursion@6.0.2:
     resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}
 
@@ -775,6 +836,9 @@ packages:
   tslib@2.8.1:
     resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==}
 
+  undici-types@7.19.2:
+    resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==}
+
   unist-util-is@6.0.1:
     resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==}
 
@@ -1063,6 +1127,33 @@ snapshots:
 
   '@opentelemetry/api@1.9.0': {}
 
+  '@protobufjs/aspromise@1.1.2': {}
+
+  '@protobufjs/base64@1.1.2': {}
+
+  '@protobufjs/codegen@2.0.5': {}
+
+  '@protobufjs/eventemitter@1.1.0': {}
+
+  '@protobufjs/fetch@1.1.0':
+    dependencies:
+      '@protobufjs/aspromise': 1.1.2
+      '@protobufjs/inquire': 1.1.1
+
+  '@protobufjs/float@1.0.2': {}
+
+  '@protobufjs/inquire@1.1.1': {}
+
+  '@protobufjs/path@1.1.2': {}
+
+  '@protobufjs/pool@1.1.0': {}
+
+  '@protobufjs/utf8@1.1.1': {}
+
+  '@ricky0123/vad-web@0.0.30':
+    dependencies:
+      onnxruntime-web: 1.25.1
+
   '@shikijs/core@3.23.0':
     dependencies:
       '@shikijs/types': 3.23.0
@@ -1162,6 +1253,10 @@ snapshots:
 
   '@types/ms@2.1.0': {}
 
+  '@types/node@25.6.0':
+    dependencies:
+      undici-types: 7.19.2
+
   '@types/trusted-types@2.0.7':
     optional: true
 
@@ -1404,10 +1499,14 @@ snapshots:
 
   fast-json-patch@3.1.1: {}
 
+  flatbuffers@25.9.23: {}
+
   get-caller-file@2.0.5: {}
 
   get-east-asian-width@1.5.0: {}
 
+  guid-typescript@1.0.9: {}
+
   hast-util-to-html@9.0.5:
     dependencies:
       '@types/hast': 3.0.4
@@ -1457,6 +1556,8 @@ snapshots:
 
   lodash-es@4.18.1: {}
 
+  long@5.3.2: {}
+
   longest-streak@3.1.0: {}
 
   magic-string@0.30.21:
@@ -1832,6 +1933,17 @@ snapshots:
       regex: 6.1.0
       regex-recursion: 6.0.2
 
+  onnxruntime-common@1.25.1: {}
+
+  onnxruntime-web@1.25.1:
+    dependencies:
+      flatbuffers: 25.9.23
+      guid-typescript: 1.0.9
+      long: 5.3.2
+      onnxruntime-common: 1.25.1
+      platform: 1.3.6
+      protobufjs: 7.5.6
+
   parse-entities@4.0.2:
     dependencies:
       '@types/unist': 2.0.11
@@ -1844,6 +1956,8 @@ snapshots:
 
   picocolors@1.1.1: {}
 
+  platform@1.3.6: {}
+
   postcss@8.5.10:
     dependencies:
       nanoid: 3.3.11
@@ -1852,6 +1966,21 @@ snapshots:
 
   property-information@7.1.0: {}
 
+  protobufjs@7.5.6:
+    dependencies:
+      '@protobufjs/aspromise': 1.1.2
+      '@protobufjs/base64': 1.1.2
+      '@protobufjs/codegen': 2.0.5
+      '@protobufjs/eventemitter': 1.1.0
+      '@protobufjs/fetch': 1.1.0
+      '@protobufjs/float': 1.0.2
+      '@protobufjs/inquire': 1.1.1
+      '@protobufjs/path': 1.1.2
+      '@protobufjs/pool': 1.1.0
+      '@protobufjs/utf8': 1.1.1
+      '@types/node': 25.6.0
+      long: 5.3.2
+
   regex-recursion@6.0.2:
     dependencies:
       regex-utilities: 2.3.0
@@ -1925,6 +2054,8 @@ snapshots:
 
   tslib@2.8.1: {}
 
+  undici-types@7.19.2: {}
+
   unist-util-is@6.0.1:
     dependencies:
       '@types/unist': 3.0.3

package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -28,7 +28,7 @@
 <script setup lang="ts">
 import { computed, onMounted, onBeforeUnmount, ref, watch } from 'vue';
 import debounce from 'lodash.debounce';
-import { requestMicAndStartVAD, stopUserMedia, getRecorder, CALIBRATION_DURATION } from './voiceActivityDetection';
+import { requestMicAndStartVAD, stopUserMedia, getRecord } from './voiceActivityDetection';
 import { Spinner } from '@/afcl'
 import { storeToRefs } from 'pinia';
 import { useAgentStore } from '../composables/useAgentStore';
@@ -41,7 +41,7 @@ const { sendAudioToServerAndHandleResponse } = agentAudio;
 const { stopGenerationAndAudio } = agentAudio;
 const { stopCurrentAudioPlayback } = agentAudio;
 const { agentAudioMode } = storeToRefs(agentAudio);
-const microphoneButtonMode = ref<'off' | 'calibrating' | 'listen' | 'transcribing' | 'generating'>('off');
+const microphoneButtonMode = ref<'off' | 'listen' | 'transcribing' | 'generating'>('off');
 const showAudioWavesAnimation = ref(false);
 const audioAmplitude = ref(0);
 const hideAnimationDebounced = debounce(() => {
@@ -93,14 +93,9 @@ function toggleChatMode() {
 }
 
 async function onStartRecording() {
-  microphoneButtonMode.value = 'calibrating';
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-  setTimeout(() => {
-    if (isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
       agentAudio.playBeep(1000);
-    }
-  }, CALIBRATION_DURATION);
 }
 
 function onStopRecording() {
@@ -145,8 +140,9 @@ function onAnySound(amplitude: number) {
 
 async function sendRecordForTranscription() {
   showAudioWavesAnimation.value = false;
-  const recordBlob = await getRecorder();
+  const recordBlob = await getRecord();
   if (recordBlob) {
+    console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
     onStopRecording();
     await sendAudioToServerAndHandleResponse(recordBlob);
     if (agentStore.isAudioChatMode) {

package/dist/custom/speech_recognition_frontend/voiceActivityDetection.ts

@@ -1,151 +1,62 @@
-import vad from 'voice-activity-detection';
-
-let currentStream: MediaStream | null = null;
-let vadInstance: any = null;
-let audioContext: AudioContext | null = null;
-let mediaRecorder: MediaRecorder | null = null;
-let recordedChunks: BlobPart[] = [];
-let wasVoiceStarted = false;
-
-export const CALIBRATION_DURATION = 1000; // in ms
+import { MicVAD, utils } from "@ricky0123/vad-web"
+
+let VADInstance: MicVAD | null = null;
+let recordedAudioChunks: Float32Array[] = [];
+let onVoiceStopCallback: () => void = () => {};
+let onVoiceStartCallback: () => void = () => {};
+let onUpdateCallback: (amplitude: number) => void = () => {};
+
+async function createVADInstance(){
+  VADInstance = await MicVAD.new({
+    onFrameProcessed: ({ isSpeech }) => {
+      onUpdateCallback(isSpeech);
+    },
+    onSpeechEnd: (audio) => {
+      recordedAudioChunks.push(audio);
+      onVoiceStopCallback();
+    },
+    onSpeechStart: () => {
+      onVoiceStartCallback();
+    },
+    onnxWASMBasePath: "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/",
+    baseAssetPath: "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/",
+  })
+}
 
 export async function requestMicAndStartVAD(
-  onVoiceStopCallback: () => void,
-  onVoiceStartCallback: () => void,
-  onUpdateCallback: (amplitude: number) => void
+  onVoiceStop: () => void,
+  onVoiceStart: () => void,
+  onUpdate: (amplitude: number) => void
 ) {
-  return new Promise<void>((resolve, reject) => {
-    try {
-      audioContext = new AudioContext();
-
-      navigator.mediaDevices
-        .getUserMedia({ audio: true })
-        .then((stream) => {
-          currentStream = stream;
-          startRecording(stream);
-          resolve();
-          startUserMedia(audioContext as AudioContext, stream, onVoiceStartCallback, onVoiceStopCallback, onUpdateCallback);
-        })
-        .catch((error) => {
-          handleMicConnectError();
-          reject(error);
-        });
-    } catch (e) {
-      handleUserMediaError();
-      reject(e);
-    }
-  });
-}
-
-function handleUserMediaError() {
-  console.error('Mic input is not supported by the browser.');
-}
-
-function handleMicConnectError() {
-  console.error('Could not connect microphone. Possible rejected by the user or is blocked by the browser.');
-}
-
-export async function stopUserMedia() {
-  wasVoiceStarted = false;
-  if (vadInstance && vadInstance.destroy) {
-    vadInstance.destroy();
-    vadInstance = null;
-  }
-
-  if (currentStream) {
-    currentStream.getTracks().forEach(track => track.stop());
-    currentStream = null;
-  }
-
-  if (audioContext) {
-    audioContext.close();
-    audioContext = null;
-  }
+  onVoiceStopCallback = onVoiceStop;
+  onVoiceStartCallback = onVoiceStart;
+  onUpdateCallback = onUpdate;
 
-  if (mediaRecorder) {
-    mediaRecorder.stop();
-    mediaRecorder = null;
+  if (!VADInstance) {
+    await createVADInstance();
   }
-
+  VADInstance?.start(); 
 }
 
-function startRecording(stream: MediaStream) {
-  recordedChunks = [];
-  mediaRecorder = new MediaRecorder(stream);
-  mediaRecorder.ondataavailable = (event: BlobEvent) => {
-    if (event.data.size > 0) {
-      recordedChunks.push(event.data);
-    }
-  };
-  mediaRecorder.start();
+export async function stopUserMedia() {
+  await VADInstance?.pause();
 }
 
-export async function getRecorder(): Promise<Blob | null> {
-  if (!mediaRecorder) {
-    return Promise.resolve(null);
+export async function getRecord() {
+  const totalSamples = recordedAudioChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  if (totalSamples === 0) {
+    return null;
   }
 
-  const recorder = mediaRecorder;
-  mediaRecorder = null;
-
-  const finalizeBlob = () => {
-    const blob = new Blob(recordedChunks, { type: recorder.mimeType || 'audio/webm' });
-    recordedChunks = [];
-    return blob;
-  };
-
-  if (recorder.state === 'inactive') {
-    return Promise.resolve(finalizeBlob());
+  const mergedAudio = new Float32Array(totalSamples);
+  let offset = 0;
+  for (const chunk of recordedAudioChunks) {
+    mergedAudio.set(chunk, offset);
+    offset += chunk.length;
   }
 
-  return new Promise<Blob>((resolve, reject) => {
-    recorder.onstop = () => {
-      resolve(finalizeBlob());
-    };
-    recorder.onerror = () => {
-      recordedChunks = [];
-      reject(new Error('Failed to finalize audio recording.'));
-    };
-    recorder.stop();
-  });
-}
-
-function startUserMedia(
-  audioContext: AudioContext, 
-  stream: MediaStream, 
-  onVoiceStartCallback: () => void,
-  onVoiceStopCallback: () => void,
-  onUpdateCallback: (amplitude: number) => void
-) {
-  const options = {
-    fftSize: 1024,
-    bufferLen: 1024,
-    smoothingTimeConstant: 0.2,
-    minCaptureFreq: 85,         // in Hz
-    maxCaptureFreq: 255,        // in Hz
-    noiseCaptureDuration: CALIBRATION_DURATION, // in ms
-    minNoiseLevel: 0.5,         // from 0 to 1
-    maxNoiseLevel: 0.7,         // from 0 to 1
-    avgNoiseMultiplier: 1.2,
-    onVoiceStart() {
-      wasVoiceStarted = true;
-      if (!mediaRecorder || mediaRecorder.state === 'inactive') {
-        startRecording(currentStream as MediaStream);
-      }
-      console.log('👹👹👹voice start👹👹👹');
-      onVoiceStartCallback();
-    },
-    onVoiceStop() {
-      if (!wasVoiceStarted) {
-        return;
-      }
-      console.log('👿👿👿voice stop👿👿👿');
-      onVoiceStopCallback();
-    }, //Doesn't work properly, so we will handle it with onUpdate callback
-    onUpdate(val: number) {
-      onUpdateCallback(val);
-    }
-  };
-
-  vadInstance = vad(audioContext, stream, options);
+  const wavBuffer = utils.encodeWAV(mergedAudio, 1, 16000, 1, 16);
+  const recordToReturn = new Blob([wavBuffer], { type: 'audio/wav' });
+  recordedAudioChunks = [];
+  return recordToReturn;
 }

package/dist/index.js

@@ -47,6 +47,15 @@ const sessionIdBodySchema = z.object({
 const createSessionBodySchema = z.object({
     triggerMessage: z.string().optional(),
 }).strict();
+function isAbortError(error) {
+    return (error instanceof DOMException && error.name === "AbortError") || (typeof error === "object" &&
+        error !== null &&
+        "name" in error &&
+        (error.name === "AbortError" || error.name === "APIUserAbortError"));
+}
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
 export default class AdminForthAgentPlugin extends AdminForthPlugin {
     parseBody(schema, body, response) {
         const parsed = schema.safeParse(body !== null && body !== void 0 ? body : {});
@@ -134,6 +143,20 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     hasAudioAdapter: Boolean(this.options.audioAdapter),
                 }
             });
+            if (!this.adminforth.config.customization.customHeadItems) {
+                this.adminforth.config.customization.customHeadItems = [];
+            }
+            this.adminforth.config.customization.customHeadItems.push({
+                tagName: 'script',
+                attributes: {
+                    src: 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/ort.wasm.min.js'
+                }
+            }, {
+                tagName: 'script',
+                attributes: {
+                    src: 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/bundle.min.js'
+                },
+            });
             if (!this.options.sessionResource) {
                 throw new Error("sessionResource is required for AdminForthAgentPlugin");
             }
@@ -151,7 +174,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
     runAgentTurn(input) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
-            var _d, _e, _f, _g;
+            var _d, _e, _f, _g, _h;
             let fullResponse = "";
             const maxTokens = (_d = this.options.maxTokens) !== null && _d !== void 0 ? _d : 1000;
             const selectedMode = (_e = this.options.modes.find((mode) => mode.name === input.modeName)) !== null && _e !== void 0 ? _e : this.options.modes[0];
@@ -172,7 +195,11 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             const modelMiddleware = primaryModelSpec.middleware;
             const userLanguage = yield detectUserLanguage(selectedMode.completionAdapter, input.prompt, input.previousUserMessages)
                 .catch((error) => {
-                logger.warn(`Failed to detect user language: ${error.message}`);
+                var _a;
+                if (((_a = input.abortSignal) === null || _a === void 0 ? void 0 : _a.aborted) || isAbortError(error)) {
+                    throw error;
+                }
+                logger.warn(`Failed to detect user language: ${getErrorMessage(error)}`);
                 return null;
             });
             const systemPrompt = buildAgentTurnSystemPrompt({
@@ -209,10 +236,13 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                 sequenceDebugSink: input.sequenceDebugCollector,
             });
             try {
-                for (var _h = true, _j = __asyncValues(stream), _k; _k = yield _j.next(), _a = _k.done, !_a; _h = true) {
-                    _c = _k.value;
-                    _h = false;
+                for (var _j = true, _k = __asyncValues(stream), _l; _l = yield _k.next(), _a = _l.done, !_a; _j = true) {
+                    _c = _l.value;
+                    _j = false;
                     const rawChunk = _c;
+                    if ((_f = input.abortSignal) === null || _f === void 0 ? void 0 : _f.aborted) {
+                        throw new DOMException("This operation was aborted", "AbortError");
+                    }
                     const [token, metadata] = rawChunk;
                     const nodeName = typeof (metadata === null || metadata === void 0 ? void 0 : metadata.langgraph_node) === "string"
                         ? metadata.langgraph_node
@@ -234,18 +264,18 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                         .map((b) => { var _a; return String((_a = b.text) !== null && _a !== void 0 ? _a : ""); })
                         .join("");
                     if (reasoningDelta) {
-                        (_f = input.emitReasoningDelta) === null || _f === void 0 ? void 0 : _f.call(input, reasoningDelta);
+                        (_g = input.emitReasoningDelta) === null || _g === void 0 ? void 0 : _g.call(input, reasoningDelta);
                     }
                     if (textDelta) {
                         fullResponse += textDelta;
-                        (_g = input.emitTextDelta) === null || _g === void 0 ? void 0 : _g.call(input, textDelta);
+                        (_h = input.emitTextDelta) === null || _h === void 0 ? void 0 : _h.call(input, textDelta);
                     }
                 }
             }
             catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_h && !_a && (_b = _j.return)) yield _b.call(_j);
+                    if (!_j && !_a && (_b = _k.return)) yield _b.call(_k);
                 }
                 finally { if (e_1) throw e_1.error; }
             }
@@ -285,14 +315,14 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                 fullResponse = agentResponse.text;
             }
             catch (error) {
-                if ((_a = input.abortSignal) === null || _a === void 0 ? void 0 : _a.aborted) {
+                if (((_a = input.abortSignal) === null || _a === void 0 ? void 0 : _a.aborted) || isAbortError(error)) {
                     aborted = true;
                     logger.info(input.abortLogMessage);
                 }
                 else {
                     failed = true;
-                    logger.error(`${input.failureLogMessage}:\n${error.message}`);
-                    fullResponse = error.message;
+                    fullResponse = getErrorMessage(error);
+                    logger.error(`${input.failureLogMessage}:\n${fullResponse}`);
                     (_b = input.emitErrorResponse) === null || _b === void 0 ? void 0 : _b.call(input, fullResponse);
                 }
             }
@@ -392,12 +422,12 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     });
                 }
                 catch (error) {
-                    if (abortSignal.aborted) {
+                    if (abortSignal.aborted || isAbortError(error)) {
                         logger.info("Agent speech transcription aborted by the client");
                         stream.end();
                         return null;
                     }
-                    logger.error(`Agent speech transcription failed:\n${error.message}`);
+                    logger.error(`Agent speech transcription failed:\n${getErrorMessage(error)}`);
                     stream.error("Speech transcription failed. Check server logs for details.");
                     stream.end();
                     return null;
@@ -481,7 +511,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     return null;
                 }
                 catch (error) {
-                    if (abortSignal.aborted) {
+                    if (abortSignal.aborted || isAbortError(error)) {
                         logger.info("Agent speech audio streaming aborted by the client");
                     }
                     else {

package/index.ts

@@ -75,6 +75,20 @@ const createSessionBodySchema = z.object({
   triggerMessage: z.string().optional(),
 }).strict();
 
+function isAbortError(error: unknown): boolean {
+  return (
+    error instanceof DOMException && error.name === "AbortError"
+  ) || (
+    typeof error === "object" &&
+    error !== null &&
+    "name" in error &&
+    (error.name === "AbortError" || error.name === "APIUserAbortError")
+  );
+}
+
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
 
 export default class AdminForthAgentPlugin extends AdminForthPlugin {
   options: PluginOptions;
@@ -176,6 +190,23 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         hasAudioAdapter: Boolean(this.options.audioAdapter),
       }
     });
+    if (!this.adminforth.config.customization.customHeadItems) {
+      this.adminforth.config.customization.customHeadItems = [];
+    }
+    this.adminforth.config.customization.customHeadItems.push(
+      {
+        tagName: 'script',
+        attributes: {
+          src: 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/ort.wasm.min.js'
+        }
+      },
+      {
+        tagName: 'script',
+        attributes: {
+          src: 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/bundle.min.js'
+        },
+      }
+    );
     if (!this.options.sessionResource) {
       throw new Error("sessionResource is required for AdminForthAgentPlugin");
     }
@@ -216,7 +247,11 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
 
     const userLanguage = await detectUserLanguage(selectedMode.completionAdapter, input.prompt, input.previousUserMessages)
       .catch((error) => {
-        logger.warn(`Failed to detect user language: ${error.message}`);
+        if (input.abortSignal?.aborted || isAbortError(error)) {
+          throw error;
+        }
+
+        logger.warn(`Failed to detect user language: ${getErrorMessage(error)}`);
         return null;
       });
     const systemPrompt = buildAgentTurnSystemPrompt({
@@ -257,6 +292,10 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
     });
 
     for await (const rawChunk of stream as AsyncIterable<[any, any]>) {
+      if (input.abortSignal?.aborted) {
+        throw new DOMException("This operation was aborted", "AbortError");
+      }
+
       const [token, metadata] = rawChunk;
 
       const nodeName =
@@ -327,13 +366,13 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
       });
       fullResponse = agentResponse.text;
     } catch (error) {
-      if (input.abortSignal?.aborted) {
+      if (input.abortSignal?.aborted || isAbortError(error)) {
         aborted = true;
         logger.info(input.abortLogMessage);
       } else {
         failed = true;
-        logger.error(`${input.failureLogMessage}:\n${error.message}`);
-        fullResponse = error.message;
+        fullResponse = getErrorMessage(error);
+        logger.error(`${input.failureLogMessage}:\n${fullResponse}`);
         input.emitErrorResponse?.(fullResponse);
       }
     }
@@ -437,13 +476,13 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             abortSignal,
           });
         } catch (error) {
-          if (abortSignal.aborted) {
+          if (abortSignal.aborted || isAbortError(error)) {
             logger.info("Agent speech transcription aborted by the client");
             stream.end();
             return null;
           }
 
-          logger.error(`Agent speech transcription failed:\n${error.message}`);
+          logger.error(`Agent speech transcription failed:\n${getErrorMessage(error)}`);
           stream.error("Speech transcription failed. Check server logs for details.");
           stream.end();
           return null;
@@ -545,7 +584,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
           stream.end();
           return null;
         } catch (error) {
-          if (abortSignal.aborted) {
+          if (abortSignal.aborted || isAbortError(error)) {
             logger.info("Agent speech audio streaming aborted by the client");
           } else {
             logger.error(`Agent speech audio streaming failed:\n${error}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.40.4",
+  "version": "1.41.1",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",