@adminforth/agent 1.43.5 → 1.43.7

package/agentResponseEvents.ts

@@ -145,12 +145,21 @@ export function createAgentEventStream(
       });
     },
 
-    audioStart(mimeType: string, format: string) {
+    audioStart(
+      mimeType: string,
+      format: string,
+      sampleRate: number,
+      channelCount: number,
+      bitsPerSample: number,
+    ) {
       stream.send({
         type: "audio-start",
         data: {
           mimeType,
           format,
+          sampleRate,
+          channelCount,
+          bitsPerSample,
         },
       });
     },

package/build.log

@@ -18,6 +18,8 @@ custom/composables/
 custom/composables/useAgentAudio.ts
 custom/composables/useAgentStore.ts
 custom/composables/useAgentTransitions.ts
+custom/composables/agentAudio/
+custom/composables/agentAudio/utils.ts
 custom/composables/agentStore/
 custom/composables/agentStore/constants.ts
 custom/composables/agentStore/pageContext.ts
@@ -58,5 +60,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
 custom/speech_recognition_frontend/types/
 custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
 
-sent 1,661,194 bytes  received 860 bytes  3,324,108.00 bytes/sec
-total size is 1,657,278  speedup is 1.00
+sent 1,663,013 bytes  received 883 bytes  3,327,792.00 bytes/sec
+total size is 1,658,998  speedup is 1.00

package/custom/composables/agentAudio/utils.ts

@@ -0,0 +1,205 @@
+const ctx = new AudioContext();
+let standbySource: AudioBufferSourceNode | null = null;
+
+const DEFAULT_PCM_SAMPLE_RATE = 24000;
+const DEFAULT_PCM_CHANNEL_COUNT = 1;
+const DEFAULT_PCM_BITS_PER_SAMPLE = 16;
+
+export type ChatResponseAudioPlayback = {
+  nextStartTime: number;
+  activeSources: Set<AudioBufferSourceNode>;
+  pendingSourceCount: number;
+  pendingPcmBytes: Uint8Array;
+  isDone: boolean;
+  isStopped: boolean;
+  sampleRate: number;
+  channelCount: number;
+  bitsPerSample: number;
+  onEnded?: () => void;
+};
+
+export async function unlockAudio() {
+  await ctx.resume();
+
+  const buffer = ctx.createBuffer(1, 1, 22050);
+  const source = ctx.createBufferSource();
+
+  source.buffer = buffer;
+  source.connect(ctx.destination);
+  source.start(0);
+}
+
+export async function startStandByAudio() {
+  const response = await fetch(
+    `/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`
+  );
+
+  const arrayBuffer = await response.arrayBuffer();
+  const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
+
+  const source = ctx.createBufferSource();
+  standbySource = source;
+
+  source.buffer = audioBuffer;
+  source.connect(ctx.destination);
+
+  source.start();
+}
+
+export function createChatResponseAudioPlayback(options: {
+  sampleRate?: number;
+  channelCount?: number;
+  bitsPerSample?: number;
+  onEnded?: () => void;
+} = {}): ChatResponseAudioPlayback {
+  return {
+    nextStartTime: ctx.currentTime,
+    activeSources: new Set(),
+    pendingSourceCount: 0,
+    pendingPcmBytes: new Uint8Array(0),
+    isDone: false,
+    isStopped: false,
+    sampleRate: options.sampleRate ?? DEFAULT_PCM_SAMPLE_RATE,
+    channelCount: options.channelCount ?? DEFAULT_PCM_CHANNEL_COUNT,
+    bitsPerSample: options.bitsPerSample ?? DEFAULT_PCM_BITS_PER_SAMPLE,
+    onEnded: options.onEnded,
+  };
+}
+
+export function playChatResponseCurrentChunks({
+  playback,
+  chunks,
+}: {
+  playback: ChatResponseAudioPlayback;
+  chunks: ArrayBuffer[];
+}) {
+  void ctx.resume().catch(() => undefined);
+
+  for (const chunk of chunks) {
+    if (playback.isStopped || !chunk.byteLength) {
+      continue;
+    }
+
+    const pcmBytes = concatUint8Arrays(playback.pendingPcmBytes, new Uint8Array(chunk));
+    const bytesPerFrame = playback.channelCount * (playback.bitsPerSample / 8);
+    const alignedByteLength = pcmBytes.byteLength - (pcmBytes.byteLength % bytesPerFrame);
+
+    playback.pendingPcmBytes = pcmBytes.slice(alignedByteLength);
+
+    if (!alignedByteLength) {
+      continue;
+    }
+
+    const audioBuffer = createAudioBufferFromPcmChunk(
+      playback,
+      pcmBytes.subarray(0, alignedByteLength)
+    );
+    const source = ctx.createBufferSource();
+    const startTime = Math.max(playback.nextStartTime, ctx.currentTime);
+
+    source.buffer = audioBuffer;
+    source.connect(ctx.destination);
+
+    playback.activeSources.add(source);
+    playback.pendingSourceCount += 1;
+    playback.nextStartTime = startTime + audioBuffer.duration;
+
+    source.addEventListener(
+      'ended',
+      () => {
+        playback.activeSources.delete(source);
+        playback.pendingSourceCount = Math.max(0, playback.pendingSourceCount - 1);
+
+        if (!playback.isStopped && playback.isDone && playback.pendingSourceCount === 0) {
+          playback.onEnded?.();
+        }
+      },
+      { once: true }
+    );
+
+    source.start(startTime);
+  }
+}
+
+export function finishChatResponseAudio(playback: ChatResponseAudioPlayback | null) {
+  if (!playback || playback.isStopped) {
+    return;
+  }
+
+  playback.isDone = true;
+
+  if (playback.pendingSourceCount === 0) {
+    playback.onEnded?.();
+  }
+}
+
+export function stopChatResponseAudio(playback: ChatResponseAudioPlayback | null) {
+  if (!playback || playback.isStopped) {
+    return;
+  }
+
+  playback.isStopped = true;
+
+  for (const source of playback.activeSources) {
+    source.stop();
+  }
+
+  playback.activeSources.clear();
+  playback.pendingSourceCount = 0;
+  playback.pendingPcmBytes = new Uint8Array(0);
+  playback.nextStartTime = ctx.currentTime;
+}
+
+function createAudioBufferFromPcmChunk(
+  playback: ChatResponseAudioPlayback,
+  chunk: Uint8Array
+) {
+  if (playback.bitsPerSample !== DEFAULT_PCM_BITS_PER_SAMPLE) {
+    throw new Error(`Unsupported PCM bit depth: ${playback.bitsPerSample}`);
+  }
+
+  const bytesPerSample = playback.bitsPerSample / 8;
+  const frameCount = chunk.byteLength / playback.channelCount / bytesPerSample;
+  const audioBuffer = ctx.createBuffer(
+    playback.channelCount,
+    frameCount,
+    playback.sampleRate
+  );
+  const pcm = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+  const channelData = Array.from(
+    { length: playback.channelCount },
+    (_, channelIndex) => audioBuffer.getChannelData(channelIndex)
+  );
+
+  for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
+    for (let channelIndex = 0; channelIndex < playback.channelCount; channelIndex += 1) {
+      const sampleOffset =
+        (frameIndex * playback.channelCount + channelIndex) * bytesPerSample;
+      const sample = pcm.getInt16(sampleOffset, true) / 32768;
+
+      channelData[channelIndex][frameIndex] = sample;
+    }
+  }
+
+  return audioBuffer;
+}
+
+function concatUint8Arrays(left: Uint8Array, right: Uint8Array) {
+  if (!left.byteLength) {
+    return right;
+  }
+
+  const combined = new Uint8Array(left.byteLength + right.byteLength);
+
+  combined.set(left, 0);
+  combined.set(right, left.byteLength);
+
+  return combined;
+}
+
+export function endStandByAudio() {
+  if (standbySource) {
+    standbySource.stop();
+    standbySource = null;
+  }
+}

package/custom/composables/useAgentAudio.ts

@@ -4,61 +4,36 @@ import { defineStore } from 'pinia';
 import type { SpeechStreamEvent } from '../types';
 import { ref } from 'vue';
 import { getCurrentPageContext } from './agentStore/pageContext';
+import {
+  createChatResponseAudioPlayback,
+  endStandByAudio,
+  finishChatResponseAudio,
+  playChatResponseCurrentChunks,
+  startStandByAudio,
+  stopChatResponseAudio,
+  unlockAudio,
+} from './agentAudio/utils';
+import type { ChatResponseAudioPlayback } from './agentAudio/utils';
 
-type StreamingAudioState = {
-  mimeType: string;
-  mediaSource: MediaSource;
-  sourceBuffer: SourceBuffer | null;
-  pendingChunks: ArrayBuffer[];
-  hasStartedPlayback: boolean;
-  isDone: boolean;
-};
-
-let standByAudio: HTMLAudioElement | null = null;
 let isStandByAudioPlaying = false;
+let isAudioUnlocked = false;
 async function playStandByAudio() {
-  if (!standByAudio) {
-    standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
-    standByAudio.addEventListener('ended', () => {
-      if (!standByAudio.paused) {
-        restartStandByAudio();
-      }
-    });
-  }
-  standByAudio.currentTime = 0;
-  await standByAudio.play();
   isStandByAudioPlaying = true;
+  await startStandByAudio();
 }
 
 function stopStandByAudio() {
-  if (!standByAudio) {
-    return;
-  }
-  standByAudio.pause();
-  standByAudio.currentTime = 0;
+  endStandByAudio();
   isStandByAudioPlaying = false;
 }
 
-function restartStandByAudio() {
-  if (standByAudio) {
-    standByAudio.currentTime = 0;
-  }
-  playStandByAudio();
-}
-
-
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
   const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
     
   let currentAbortController: AbortController | null = null;
-  let isPlaying = false;
-  let currentAudio: HTMLAudioElement | null = null;
-  let currentAudioObjectUrl: string | null = null;
-  let currentStreamingAudio: StreamingAudioState | null = null;
-  let bufferedAudioChunks: ArrayBuffer[] = [];
-  let bufferedAudioMimeType = 'audio/mpeg';
+  let currentStreamingAudio: ChatResponseAudioPlayback | null = null;
   let wasAudioResponseReceived = false;
 
   function stopGenerationAndAudio() {
@@ -72,6 +47,10 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   }
 
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
+    if (!isAudioUnlocked) {
+      await unlockAudio();
+      isAudioUnlocked = true;
+    }
     currentAbortController = new AbortController();
     wasAudioResponseReceived = false;
     const formData = new FormData();
@@ -192,7 +171,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       wasAudioResponseReceived = true;
       isStreamingResponse.value = false;
       agentAudioMode.value = 'fetchingAudio';
-      initializeAudioStream(event.data.mimeType);
+      initializeAudioStream(event.data);
       agentAudioMode.value = 'playingAgentResponse';
       return;
     }
@@ -215,162 +194,39 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
   }
 
-  async function setIsPlaying(value: boolean) {
-    isPlaying = value;
-
-    if (!currentAudio) {
-      return;
-    }
-
-    if (!isPlaying) {
-      currentAudio.pause();
-      currentAudio.currentTime = 0;
-      return;
-    }
-    agentAudioMode.value = 'playingAgentResponse';
-    await void currentAudio.play().catch((error) => {
-      console.error('Failed to play audio:', error);
-    });
-  }
-
-  function initializeAudioStream(mimeType: string) {
+  function initializeAudioStream(audioData: Extract<SpeechStreamEvent, { type: 'audio-start' }>['data']) {
     stopCurrentAudioPlayback();
-    bufferedAudioMimeType = mimeType;
-
-    if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(mimeType)) {
-      return;
-    }
-
-    const mediaSource = new MediaSource();
-    currentAudioObjectUrl = URL.createObjectURL(mediaSource);
-    currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    currentStreamingAudio = {
-      mimeType,
-      mediaSource,
-      sourceBuffer: null,
-      pendingChunks: [],
-      hasStartedPlayback: false,
-      isDone: false,
-    };
-
-    mediaSource.addEventListener('sourceopen', handleMediaSourceOpen, { once: true });
-  }
-
-  function handleMediaSourceOpen() {
-    if (!currentStreamingAudio) {
-      return;
-    }
-
-    try {
-      currentStreamingAudio.sourceBuffer = currentStreamingAudio.mediaSource.addSourceBuffer(currentStreamingAudio.mimeType);
-      currentStreamingAudio.sourceBuffer.mode = 'sequence';
-      currentStreamingAudio.sourceBuffer.addEventListener('updateend', flushStreamingAudioQueue);
-      flushStreamingAudioQueue();
-    } catch (error) {
-      console.error('Failed to initialize streaming audio playback:', error);
-      bufferedAudioChunks.push(...currentStreamingAudio.pendingChunks);
-      detachStreamingAudio();
-      destroyCurrentAudioElement();
-    }
+    currentStreamingAudio = createChatResponseAudioPlayback({
+      sampleRate: audioData.sampleRate,
+      channelCount: audioData.channelCount,
+      bitsPerSample: audioData.bitsPerSample,
+      onEnded: handleAudioEnded,
+    });
   }
 
   function appendAudioChunk(base64: string) {
-    const chunk = base64ToArrayBuffer(base64);
-
-    if (!currentStreamingAudio) {
-      bufferedAudioChunks.push(chunk);
-      return;
-    }
-
-    currentStreamingAudio.pendingChunks.push(chunk);
-    flushStreamingAudioQueue();
-  }
-
-  function flushStreamingAudioQueue() {
-    if (!currentStreamingAudio?.sourceBuffer || currentStreamingAudio.sourceBuffer.updating) {
-      return;
-    }
-
-    const nextChunk = currentStreamingAudio.pendingChunks.shift();
-
-    if (nextChunk) {
-      currentStreamingAudio.sourceBuffer.appendBuffer(nextChunk);
-
-      if (!currentStreamingAudio.hasStartedPlayback) {
-        currentStreamingAudio.hasStartedPlayback = true;
-        setIsPlaying(true);
-      }
-
-      return;
-    }
-
-    if (currentStreamingAudio.isDone && currentStreamingAudio.mediaSource.readyState === 'open') {
-      currentStreamingAudio.mediaSource.endOfStream();
-    }
+    playChatResponseCurrentChunks({
+      playback: currentStreamingAudio!,
+      chunks: [base64ToArrayBuffer(base64)],
+    });
   }
 
   function finishAudioStream() {
-    if (currentStreamingAudio) {
-      currentStreamingAudio.isDone = true;
-      flushStreamingAudioQueue();
-      return;
-    }
-
-    if (!bufferedAudioChunks.length) {
-      return;
-    }
-
-    playAudioChunks(bufferedAudioChunks, bufferedAudioMimeType);
-    bufferedAudioChunks = [];
-  }
-
-  function detachStreamingAudio() {
-    if (currentStreamingAudio?.sourceBuffer) {
-      currentStreamingAudio.sourceBuffer.removeEventListener('updateend', flushStreamingAudioQueue);
-    }
-
-    currentStreamingAudio = null;
-  }
-
-  function destroyCurrentAudioElement() {
-    if (currentAudio) {
-      currentAudio.pause();
-      currentAudio.currentTime = 0;
-      currentAudio.src = '';
-      currentAudio.load();
-      currentAudio = null;
-    }
-
-    if (currentAudioObjectUrl) {
-      URL.revokeObjectURL(currentAudioObjectUrl);
-      currentAudioObjectUrl = null;
-    }
-
-    isPlaying = false;
+    finishChatResponseAudio(currentStreamingAudio);
   }
 
   function stopCurrentAudioPlayback(dontResetMode = false) {
     stopStandByAudio();
-    bufferedAudioChunks = [];
-    bufferedAudioMimeType = 'audio/mpeg';
-    detachStreamingAudio();
-    destroyCurrentAudioElement();
+    stopChatResponseAudio(currentStreamingAudio);
+    currentStreamingAudio = null;
     if (!dontResetMode) {
       setAudioModeReadyToRespond();
     }
   }
 
   function handleAudioEnded() {
+    currentStreamingAudio = null;
     setAudioModeReadyToRespond();
-    stopCurrentAudioPlayback();
-  }
-
-  function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
-    currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
-    currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    setIsPlaying(true);
   }
 
   function base64ToArrayBuffer(base64: string) {

package/custom/conversation_area/TextRenderer.vue

@@ -1,9 +1,9 @@
 <template>
   <div
-    class="flex rounded-xl border border-gray-200 dark:border-gray-700"
+    class="flex rounded-xl border px-4 border-gray-200 dark:border-gray-700"
     @click="handleMarkdownLinkClick"
     :class="[
-      hasVegaLite ? 'w-full px-6 my-2' : 'px-4 m-2',
+      hasVegaLite ? 'w-full my-2' : 'm-2',
       props.role === 'user' ? 'bg-lightListTableHeading dark:bg-darkListTableHeading self-end max-w-[80%] mr-4' 
         : 'border-none self-start'
     ]"

package/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -1,7 +1,7 @@
 <template>
   <div 
     class="absolute bottom-2 flex items-center justify-center z-10 gap-4"
-    :class="[!agentStore.isAudioChatMode ? 'right-16': agentStore.isMobile ? 'right-1/2 translate-x-1/2' : 'right-1/2 translate-x-3/4']"
+    :class="[!agentStore.isAudioChatMode ? 'right-16': agentStore.isMobile ? 'right-1/2 translate-x-1/2' : 'right-1/2 translate-x-1/2']"
   > 
     <button 
       v-if="isAudioChatMode && microphoneButtonMode === 'generating'"
@@ -9,9 +9,10 @@
       @click="stopCurrentGeneration()"
     >
       <div class="w-3 h-3 bg-white rounded-full"/>
-      {{ $t('Break') }}
+      {{ $t('Stop') }}
     </button>
-    <button 
+    <button
+      v-else 
       class="h-9 bg-lightPrimary dark:bg-darkPrimary 
         hover:opacity-90 rounded-full flex items-center justify-center
         transition-all duration-300 ease-in-out overflow-hidden"
@@ -89,6 +90,7 @@ watch(agentAudioMode, async (newVal) => {
     if(isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
       await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
+      agentAudio.playBeep(1000);
     } else {
       microphoneButtonMode.value = 'off';
     }

package/custom/types.ts

@@ -74,6 +74,9 @@ export type SpeechStreamEvent =
       data: {
         mimeType: string;
         format: string;
+        sampleRate: number;
+        channelCount: number;
+        bitsPerSample: number;
       };
     }
   | {

package/dist/agentResponseEvents.js

@@ -96,12 +96,15 @@ export function createAgentEventStream(res, options = {}) {
                 },
             });
         },
-        audioStart(mimeType, format) {
+        audioStart(mimeType, format, sampleRate, channelCount, bitsPerSample) {
             stream.send({
                 type: "audio-start",
                 data: {
                     mimeType,
                     format,
+                    sampleRate,
+                    channelCount,
+                    bitsPerSample,
                 },
             });
         },

package/dist/custom/composables/agentAudio/utils.ts

@@ -0,0 +1,205 @@
+const ctx = new AudioContext();
+let standbySource: AudioBufferSourceNode | null = null;
+
+const DEFAULT_PCM_SAMPLE_RATE = 24000;
+const DEFAULT_PCM_CHANNEL_COUNT = 1;
+const DEFAULT_PCM_BITS_PER_SAMPLE = 16;
+
+export type ChatResponseAudioPlayback = {
+  nextStartTime: number;
+  activeSources: Set<AudioBufferSourceNode>;
+  pendingSourceCount: number;
+  pendingPcmBytes: Uint8Array;
+  isDone: boolean;
+  isStopped: boolean;
+  sampleRate: number;
+  channelCount: number;
+  bitsPerSample: number;
+  onEnded?: () => void;
+};
+
+export async function unlockAudio() {
+  await ctx.resume();
+
+  const buffer = ctx.createBuffer(1, 1, 22050);
+  const source = ctx.createBufferSource();
+
+  source.buffer = buffer;
+  source.connect(ctx.destination);
+  source.start(0);
+}
+
+export async function startStandByAudio() {
+  const response = await fetch(
+    `/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`
+  );
+
+  const arrayBuffer = await response.arrayBuffer();
+  const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
+
+  const source = ctx.createBufferSource();
+  standbySource = source;
+
+  source.buffer = audioBuffer;
+  source.connect(ctx.destination);
+
+  source.start();
+}
+
+export function createChatResponseAudioPlayback(options: {
+  sampleRate?: number;
+  channelCount?: number;
+  bitsPerSample?: number;
+  onEnded?: () => void;
+} = {}): ChatResponseAudioPlayback {
+  return {
+    nextStartTime: ctx.currentTime,
+    activeSources: new Set(),
+    pendingSourceCount: 0,
+    pendingPcmBytes: new Uint8Array(0),
+    isDone: false,
+    isStopped: false,
+    sampleRate: options.sampleRate ?? DEFAULT_PCM_SAMPLE_RATE,
+    channelCount: options.channelCount ?? DEFAULT_PCM_CHANNEL_COUNT,
+    bitsPerSample: options.bitsPerSample ?? DEFAULT_PCM_BITS_PER_SAMPLE,
+    onEnded: options.onEnded,
+  };
+}
+
+export function playChatResponseCurrentChunks({
+  playback,
+  chunks,
+}: {
+  playback: ChatResponseAudioPlayback;
+  chunks: ArrayBuffer[];
+}) {
+  void ctx.resume().catch(() => undefined);
+
+  for (const chunk of chunks) {
+    if (playback.isStopped || !chunk.byteLength) {
+      continue;
+    }
+
+    const pcmBytes = concatUint8Arrays(playback.pendingPcmBytes, new Uint8Array(chunk));
+    const bytesPerFrame = playback.channelCount * (playback.bitsPerSample / 8);
+    const alignedByteLength = pcmBytes.byteLength - (pcmBytes.byteLength % bytesPerFrame);
+
+    playback.pendingPcmBytes = pcmBytes.slice(alignedByteLength);
+
+    if (!alignedByteLength) {
+      continue;
+    }
+
+    const audioBuffer = createAudioBufferFromPcmChunk(
+      playback,
+      pcmBytes.subarray(0, alignedByteLength)
+    );
+    const source = ctx.createBufferSource();
+    const startTime = Math.max(playback.nextStartTime, ctx.currentTime);
+
+    source.buffer = audioBuffer;
+    source.connect(ctx.destination);
+
+    playback.activeSources.add(source);
+    playback.pendingSourceCount += 1;
+    playback.nextStartTime = startTime + audioBuffer.duration;
+
+    source.addEventListener(
+      'ended',
+      () => {
+        playback.activeSources.delete(source);
+        playback.pendingSourceCount = Math.max(0, playback.pendingSourceCount - 1);
+
+        if (!playback.isStopped && playback.isDone && playback.pendingSourceCount === 0) {
+          playback.onEnded?.();
+        }
+      },
+      { once: true }
+    );
+
+    source.start(startTime);
+  }
+}
+
+export function finishChatResponseAudio(playback: ChatResponseAudioPlayback | null) {
+  if (!playback || playback.isStopped) {
+    return;
+  }
+
+  playback.isDone = true;
+
+  if (playback.pendingSourceCount === 0) {
+    playback.onEnded?.();
+  }
+}
+
+export function stopChatResponseAudio(playback: ChatResponseAudioPlayback | null) {
+  if (!playback || playback.isStopped) {
+    return;
+  }
+
+  playback.isStopped = true;
+
+  for (const source of playback.activeSources) {
+    source.stop();
+  }
+
+  playback.activeSources.clear();
+  playback.pendingSourceCount = 0;
+  playback.pendingPcmBytes = new Uint8Array(0);
+  playback.nextStartTime = ctx.currentTime;
+}
+
+function createAudioBufferFromPcmChunk(
+  playback: ChatResponseAudioPlayback,
+  chunk: Uint8Array
+) {
+  if (playback.bitsPerSample !== DEFAULT_PCM_BITS_PER_SAMPLE) {
+    throw new Error(`Unsupported PCM bit depth: ${playback.bitsPerSample}`);
+  }
+
+  const bytesPerSample = playback.bitsPerSample / 8;
+  const frameCount = chunk.byteLength / playback.channelCount / bytesPerSample;
+  const audioBuffer = ctx.createBuffer(
+    playback.channelCount,
+    frameCount,
+    playback.sampleRate
+  );
+  const pcm = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+  const channelData = Array.from(
+    { length: playback.channelCount },
+    (_, channelIndex) => audioBuffer.getChannelData(channelIndex)
+  );
+
+  for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
+    for (let channelIndex = 0; channelIndex < playback.channelCount; channelIndex += 1) {
+      const sampleOffset =
+        (frameIndex * playback.channelCount + channelIndex) * bytesPerSample;
+      const sample = pcm.getInt16(sampleOffset, true) / 32768;
+
+      channelData[channelIndex][frameIndex] = sample;
+    }
+  }
+
+  return audioBuffer;
+}
+
+function concatUint8Arrays(left: Uint8Array, right: Uint8Array) {
+  if (!left.byteLength) {
+    return right;
+  }
+
+  const combined = new Uint8Array(left.byteLength + right.byteLength);
+
+  combined.set(left, 0);
+  combined.set(right, left.byteLength);
+
+  return combined;
+}
+
+export function endStandByAudio() {
+  if (standbySource) {
+    standbySource.stop();
+    standbySource = null;
+  }
+}

package/dist/custom/composables/useAgentAudio.ts

@@ -4,61 +4,36 @@ import { defineStore } from 'pinia';
 import type { SpeechStreamEvent } from '../types';
 import { ref } from 'vue';
 import { getCurrentPageContext } from './agentStore/pageContext';
+import {
+  createChatResponseAudioPlayback,
+  endStandByAudio,
+  finishChatResponseAudio,
+  playChatResponseCurrentChunks,
+  startStandByAudio,
+  stopChatResponseAudio,
+  unlockAudio,
+} from './agentAudio/utils';
+import type { ChatResponseAudioPlayback } from './agentAudio/utils';
 
-type StreamingAudioState = {
-  mimeType: string;
-  mediaSource: MediaSource;
-  sourceBuffer: SourceBuffer | null;
-  pendingChunks: ArrayBuffer[];
-  hasStartedPlayback: boolean;
-  isDone: boolean;
-};
-
-let standByAudio: HTMLAudioElement | null = null;
 let isStandByAudioPlaying = false;
+let isAudioUnlocked = false;
 async function playStandByAudio() {
-  if (!standByAudio) {
-    standByAudio = new Audio(`/plugins/AdminForthAgentPlugin/agentAudio/agent-processing.mp3`);
-    standByAudio.addEventListener('ended', () => {
-      if (!standByAudio.paused) {
-        restartStandByAudio();
-      }
-    });
-  }
-  standByAudio.currentTime = 0;
-  await standByAudio.play();
   isStandByAudioPlaying = true;
+  await startStandByAudio();
 }
 
 function stopStandByAudio() {
-  if (!standByAudio) {
-    return;
-  }
-  standByAudio.pause();
-  standByAudio.currentTime = 0;
+  endStandByAudio();
   isStandByAudioPlaying = false;
 }
 
-function restartStandByAudio() {
-  if (standByAudio) {
-    standByAudio.currentTime = 0;
-  }
-  playStandByAudio();
-}
-
-
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
   const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
     
   let currentAbortController: AbortController | null = null;
-  let isPlaying = false;
-  let currentAudio: HTMLAudioElement | null = null;
-  let currentAudioObjectUrl: string | null = null;
-  let currentStreamingAudio: StreamingAudioState | null = null;
-  let bufferedAudioChunks: ArrayBuffer[] = [];
-  let bufferedAudioMimeType = 'audio/mpeg';
+  let currentStreamingAudio: ChatResponseAudioPlayback | null = null;
   let wasAudioResponseReceived = false;
 
   function stopGenerationAndAudio() {
@@ -72,6 +47,10 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   }
 
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
+    if (!isAudioUnlocked) {
+      await unlockAudio();
+      isAudioUnlocked = true;
+    }
     currentAbortController = new AbortController();
     wasAudioResponseReceived = false;
     const formData = new FormData();
@@ -192,7 +171,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       wasAudioResponseReceived = true;
       isStreamingResponse.value = false;
       agentAudioMode.value = 'fetchingAudio';
-      initializeAudioStream(event.data.mimeType);
+      initializeAudioStream(event.data);
       agentAudioMode.value = 'playingAgentResponse';
       return;
     }
@@ -215,162 +194,39 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
   }
 
-  async function setIsPlaying(value: boolean) {
-    isPlaying = value;
-
-    if (!currentAudio) {
-      return;
-    }
-
-    if (!isPlaying) {
-      currentAudio.pause();
-      currentAudio.currentTime = 0;
-      return;
-    }
-    agentAudioMode.value = 'playingAgentResponse';
-    await void currentAudio.play().catch((error) => {
-      console.error('Failed to play audio:', error);
-    });
-  }
-
-  function initializeAudioStream(mimeType: string) {
+  function initializeAudioStream(audioData: Extract<SpeechStreamEvent, { type: 'audio-start' }>['data']) {
     stopCurrentAudioPlayback();
-    bufferedAudioMimeType = mimeType;
-
-    if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(mimeType)) {
-      return;
-    }
-
-    const mediaSource = new MediaSource();
-    currentAudioObjectUrl = URL.createObjectURL(mediaSource);
-    currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    currentStreamingAudio = {
-      mimeType,
-      mediaSource,
-      sourceBuffer: null,
-      pendingChunks: [],
-      hasStartedPlayback: false,
-      isDone: false,
-    };
-
-    mediaSource.addEventListener('sourceopen', handleMediaSourceOpen, { once: true });
-  }
-
-  function handleMediaSourceOpen() {
-    if (!currentStreamingAudio) {
-      return;
-    }
-
-    try {
-      currentStreamingAudio.sourceBuffer = currentStreamingAudio.mediaSource.addSourceBuffer(currentStreamingAudio.mimeType);
-      currentStreamingAudio.sourceBuffer.mode = 'sequence';
-      currentStreamingAudio.sourceBuffer.addEventListener('updateend', flushStreamingAudioQueue);
-      flushStreamingAudioQueue();
-    } catch (error) {
-      console.error('Failed to initialize streaming audio playback:', error);
-      bufferedAudioChunks.push(...currentStreamingAudio.pendingChunks);
-      detachStreamingAudio();
-      destroyCurrentAudioElement();
-    }
+    currentStreamingAudio = createChatResponseAudioPlayback({
+      sampleRate: audioData.sampleRate,
+      channelCount: audioData.channelCount,
+      bitsPerSample: audioData.bitsPerSample,
+      onEnded: handleAudioEnded,
+    });
   }
 
   function appendAudioChunk(base64: string) {
-    const chunk = base64ToArrayBuffer(base64);
-
-    if (!currentStreamingAudio) {
-      bufferedAudioChunks.push(chunk);
-      return;
-    }
-
-    currentStreamingAudio.pendingChunks.push(chunk);
-    flushStreamingAudioQueue();
-  }
-
-  function flushStreamingAudioQueue() {
-    if (!currentStreamingAudio?.sourceBuffer || currentStreamingAudio.sourceBuffer.updating) {
-      return;
-    }
-
-    const nextChunk = currentStreamingAudio.pendingChunks.shift();
-
-    if (nextChunk) {
-      currentStreamingAudio.sourceBuffer.appendBuffer(nextChunk);
-
-      if (!currentStreamingAudio.hasStartedPlayback) {
-        currentStreamingAudio.hasStartedPlayback = true;
-        setIsPlaying(true);
-      }
-
-      return;
-    }
-
-    if (currentStreamingAudio.isDone && currentStreamingAudio.mediaSource.readyState === 'open') {
-      currentStreamingAudio.mediaSource.endOfStream();
-    }
+    playChatResponseCurrentChunks({
+      playback: currentStreamingAudio!,
+      chunks: [base64ToArrayBuffer(base64)],
+    });
   }
 
   function finishAudioStream() {
-    if (currentStreamingAudio) {
-      currentStreamingAudio.isDone = true;
-      flushStreamingAudioQueue();
-      return;
-    }
-
-    if (!bufferedAudioChunks.length) {
-      return;
-    }
-
-    playAudioChunks(bufferedAudioChunks, bufferedAudioMimeType);
-    bufferedAudioChunks = [];
-  }
-
-  function detachStreamingAudio() {
-    if (currentStreamingAudio?.sourceBuffer) {
-      currentStreamingAudio.sourceBuffer.removeEventListener('updateend', flushStreamingAudioQueue);
-    }
-
-    currentStreamingAudio = null;
-  }
-
-  function destroyCurrentAudioElement() {
-    if (currentAudio) {
-      currentAudio.pause();
-      currentAudio.currentTime = 0;
-      currentAudio.src = '';
-      currentAudio.load();
-      currentAudio = null;
-    }
-
-    if (currentAudioObjectUrl) {
-      URL.revokeObjectURL(currentAudioObjectUrl);
-      currentAudioObjectUrl = null;
-    }
-
-    isPlaying = false;
+    finishChatResponseAudio(currentStreamingAudio);
   }
 
   function stopCurrentAudioPlayback(dontResetMode = false) {
     stopStandByAudio();
-    bufferedAudioChunks = [];
-    bufferedAudioMimeType = 'audio/mpeg';
-    detachStreamingAudio();
-    destroyCurrentAudioElement();
+    stopChatResponseAudio(currentStreamingAudio);
+    currentStreamingAudio = null;
     if (!dontResetMode) {
       setAudioModeReadyToRespond();
     }
   }
 
   function handleAudioEnded() {
+    currentStreamingAudio = null;
     setAudioModeReadyToRespond();
-    stopCurrentAudioPlayback();
-  }
-
-  function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
-    currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
-    currentAudio = new Audio(currentAudioObjectUrl);
-    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
-    setIsPlaying(true);
   }
 
   function base64ToArrayBuffer(base64: string) {

package/dist/custom/conversation_area/TextRenderer.vue

@@ -1,9 +1,9 @@
 <template>
   <div
-    class="flex rounded-xl border border-gray-200 dark:border-gray-700"
+    class="flex rounded-xl border px-4 border-gray-200 dark:border-gray-700"
     @click="handleMarkdownLinkClick"
     :class="[
-      hasVegaLite ? 'w-full px-6 my-2' : 'px-4 m-2',
+      hasVegaLite ? 'w-full my-2' : 'm-2',
       props.role === 'user' ? 'bg-lightListTableHeading dark:bg-darkListTableHeading self-end max-w-[80%] mr-4' 
         : 'border-none self-start'
     ]"

package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -1,7 +1,7 @@
 <template>
   <div 
     class="absolute bottom-2 flex items-center justify-center z-10 gap-4"
-    :class="[!agentStore.isAudioChatMode ? 'right-16': agentStore.isMobile ? 'right-1/2 translate-x-1/2' : 'right-1/2 translate-x-3/4']"
+    :class="[!agentStore.isAudioChatMode ? 'right-16': agentStore.isMobile ? 'right-1/2 translate-x-1/2' : 'right-1/2 translate-x-1/2']"
   > 
     <button 
       v-if="isAudioChatMode && microphoneButtonMode === 'generating'"
@@ -9,9 +9,10 @@
       @click="stopCurrentGeneration()"
     >
       <div class="w-3 h-3 bg-white rounded-full"/>
-      {{ $t('Break') }}
+      {{ $t('Stop') }}
     </button>
-    <button 
+    <button
+      v-else 
       class="h-9 bg-lightPrimary dark:bg-darkPrimary 
         hover:opacity-90 rounded-full flex items-center justify-center
         transition-all duration-300 ease-in-out overflow-hidden"
@@ -89,6 +90,7 @@ watch(agentAudioMode, async (newVal) => {
     if(isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
       await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
+      agentAudio.playBeep(1000);
     } else {
       microphoneButtonMode.value = 'off';
     }

package/dist/custom/types.ts

@@ -74,6 +74,9 @@ export type SpeechStreamEvent =
       data: {
         mimeType: string;
         format: string;
+        sampleRate: number;
+        channelCount: number;
+        bitsPerSample: number;
       };
     }
   | {

package/dist/index.js

@@ -478,10 +478,10 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                         text: sanitizeSpeechText(agentResponse.text),
                         stream: true,
                         streamFormat: "audio",
-                        format: "mp3",
+                        format: "pcm",
                         abortSignal,
                     });
-                    stream.audioStart(speech.mimeType, speech.format);
+                    stream.audioStart(speech.mimeType, speech.format, 24000, 1, 16);
                     const reader = speech.audioStream.getReader();
                     const cancelAudioStream = () => {
                         void reader.cancel().catch(() => undefined);

package/index.ts

@@ -544,11 +544,11 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             text: sanitizeSpeechText(agentResponse.text),
             stream: true,
             streamFormat: "audio",
-            format: "mp3",
+            format: "pcm",
             abortSignal,
           });
 
-          stream.audioStart(speech.mimeType, speech.format);
+          stream.audioStart(speech.mimeType, speech.format, 24000, 1, 16);
 
           const reader = speech.audioStream.getReader();
           const cancelAudioStream = () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.43.5",
+  "version": "1.43.7",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",