@adminforth/agent 1.37.0 → 1.38.0

package/dist/custom/speech_recognition_frontend/AudioLines.vue

@@ -0,0 +1,97 @@
+<template> 
+  <div 
+    class=" bg-white w-[0.2rem] rounded-sm transition-all duration-300 ease-in-out"
+    :class="{
+      'recordingAnimation1' : showAnimation,
+      'h-2': !isRecording,
+      'h-1': isRecording,
+    }"  
+  />
+  <div 
+    class=" bg-white w-[0.2rem] rounded-sm transition-all duration-300 ease-in-out"
+    :class="{
+      'recordingAnimation2' : showAnimation,
+      'h-4': !isRecording,
+      'h-1': isRecording,
+    }"  
+  />
+  <div 
+    class=" bg-white w-[0.2rem] rounded-sm transition-all duration-300 ease-in-out"
+    :class="{
+      'recordingAnimation3' : showAnimation,
+      'h-3': !isRecording,
+      'h-1': isRecording,
+    }"  
+  />
+  <div 
+    class=" bg-white w-[0.2rem] rounded-sm transition-all duration-300 ease-in-out"
+    :class="{
+      'recordingAnimation4' : showAnimation,
+      'h-2': !isRecording,
+      'h-1': isRecording,
+    }"  
+  />
+  <template v-if="isRecording">
+    <div 
+      class=" bg-white w-[0.2rem] rounded-sm h-1 transition-all duration-300 ease-in-out"
+      :class="{
+        'recordingAnimation5' : showAnimation,
+      }"  
+    />
+    <p class="text-white ml-2">End</p>
+  </template>
+</template>
+
+
+
+<script setup lang="ts">
+
+const props = defineProps<{
+  showAnimation: boolean;
+  isRecording: boolean;
+}>();
+
+</script>
+
+<style scoped lang="scss">
+  .recordingAnimation1 {
+    animation: recordingAnimation 1s infinite;
+    height: 0.3rem;
+  }
+
+  .recordingAnimation2 {
+    animation: recordingAnimation 1s infinite;
+    animation-delay: 0.2s;
+    height: 0.5rem;
+  }
+
+  .recordingAnimation3 {
+    animation: recordingAnimation 1s infinite;
+    animation-delay: 0.4s;
+    height: 0.4rem;
+  }
+  
+  .recordingAnimation4 {
+    animation: recordingAnimation 1s infinite;
+    animation-delay: 0.6s;
+    height: 0.5rem;
+  }
+
+  .recordingAnimation5 {
+    animation: recordingAnimation 1s infinite;
+    animation-delay: 0.8s;
+    height: 0.3rem;
+  }
+
+  @keyframes recordingAnimation {
+    0% {
+      transform: scaleY(1);
+    }
+    50% {
+      transform: scaleY(2);
+    }
+    100% {
+      transform: scaleY(1);
+    }
+  }
+</style>

package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -0,0 +1,157 @@
+<template>
+  <button 
+    class="absolute bottom-2 h-9 bg-lightPrimary dark:bg-darkPrimary 
+      hover:opacity-90 rounded-full flex items-center justify-center
+      transition-all duration-300 ease-in-out overflow-hidden"
+    :class="[isAudioChatMode ? 'w-32 px-2': 'w-9', !agentStore.isAudioChatMode ? 'right-16': 'right-1/2 translate-x-1/2']" 
+    @click="toggleChatMode"
+  >
+    <div class="w-5 h-5 flex items-center justify-center">
+      <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
+        <AudioLines :showAnimation="showAudioWavesAnimation" :isRecording="microphoneButtonMode === 'listen'" />
+      </div>
+      <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
+        <span class="w-3 h-3 bg-white rounded-sm" />
+        {{ $t('Stop') }}
+      </div>
+      <Spinner v-else class="w-4 h-4 text-lightButtonsText dark:text-darkButtonsText fill-lightButtonsBackground dark:fill-darkPrimary" />
+    </div>
+   
+  </button>
+</template>
+
+
+<script setup lang="ts">
+import { computed, onMounted, onBeforeUnmount, ref, watch } from 'vue';
+import debounce from 'lodash/debounce';
+import { requestMicAndStartVAD, stopUserMedia, getRecorder, CALIBRATION_DURATION } from './voiceActivityDetection';
+import { Spinner } from '@/afcl'
+import { storeToRefs } from 'pinia';
+import { useAgentStore } from '../composables/useAgentStore';
+import { useAgentAudio } from '../composables/useAgentAudio';
+import AudioLines from './AudioLines.vue';
+
+const agentStore = useAgentStore();
+const agentAudio = useAgentAudio();
+const { sendAudioToServerAndHandleResponse } = agentAudio;
+const { stopGenerationAndAudio } = agentAudio;
+const { stopCurrentAudioPlayback } = agentAudio;
+const { agentAudioMode } = storeToRefs(agentAudio);
+const microphoneButtonMode = ref<'off' | 'calibrating' | 'listen' | 'transcribing' | 'generating'>('off');
+const showAudioWavesAnimation = ref(false);
+const hideAnimationDebounced = debounce(() => {
+  showAudioWavesAnimation.value = false;
+}, 100);
+const sendUserRecordDebounced = debounce(() => {
+  sendRecordForTranscription();
+}, 500);
+
+const isAudioChatMode = computed(() => agentStore.isAudioChatMode);
+
+onMounted(() => {
+  agentStore.registerOnBeforeChatCloseCallback(async () => {
+    if(agentStore.isAudioChatMode) {
+      onStopRecording();
+      resetAll();
+      agentStore.setIsAudioChatMode(false);
+    }
+  });
+});
+
+watch(agentAudioMode, (newVal) => {
+  if(newVal === 'streaming') {
+    stopCurrentAudioPlayback(true);
+    microphoneButtonMode.value = 'generating';
+  } else if (newVal === 'transcribing') {
+    microphoneButtonMode.value = 'transcribing';
+  } else if (newVal === 'fetchingAudio') {
+    //Generation is done, waiting for audio to be ready
+  } else if (newVal === 'playingAgentResponse') {
+    // Audio is playing
+  } else {
+    if(isAudioChatMode.value) {
+      microphoneButtonMode.value = 'listen';
+    } else {
+      microphoneButtonMode.value = 'off';
+    }
+  }
+})
+
+function toggleChatMode() {
+  agentStore.setIsAudioChatMode(!isAudioChatMode.value);
+  if (isAudioChatMode.value) {
+    onStartRecording();
+  } else {
+    resetAll();
+    onStopRecording();
+  }
+}
+
+async function onStartRecording() {
+  microphoneButtonMode.value = 'calibrating';
+  await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
+  setTimeout(() => {
+    microphoneButtonMode.value = 'listen';
+    agentAudio.playBeep(1000);
+  }, CALIBRATION_DURATION);
+}
+
+function onStopRecording() {
+  agentAudio.playBeep(600);
+  stopUserMedia();
+  showAudioWavesAnimation.value = false;
+}
+
+function resetAll() {
+  stopGenerationAndAudio();
+  microphoneButtonMode.value = 'off';
+  showAudioWavesAnimation.value = false;
+  hideAnimationDebounced.cancel();
+  sendUserRecordDebounced.cancel();
+}
+
+
+function saidSomething() {
+  showAudioWavesAnimation.value = true;
+  hideAnimationDebounced();
+  sendUserRecordDebounced();
+}
+
+function stopRecording() {
+  hideAnimationDebounced.cancel();
+  sendUserRecordDebounced.cancel();
+}
+
+function onAnySound(amplitude: number) {
+  if(amplitude < 0.01) {
+    showAudioWavesAnimation.value = false;
+    return;
+  }
+  showAudioWavesAnimation.value = true;
+  hideAnimationDebounced.cancel();
+}
+
+async function sendRecordForTranscription() {
+  showAudioWavesAnimation.value = false;
+  const recordBlob = await getRecorder();
+  if (recordBlob) {
+    onStopRecording();
+    await sendAudioToServerAndHandleResponse(recordBlob);
+    if (agentStore.isAudioChatMode) {
+      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
+    }
+  } else { 
+    console.error('No audio recorded');
+  }
+}
+
+onBeforeUnmount(() => {
+  stopUserMedia();
+  agentStore.setIsAudioChatMode(false);
+  onStopRecording();
+  hideAnimationDebounced.cancel();
+  sendUserRecordDebounced.cancel();
+});
+
+
+</script>

package/dist/custom/speech_recognition_frontend/types/voice-activity-detection.d.ts

@@ -0,0 +1,22 @@
+declare module 'voice-activity-detection' {
+  type VADOptions = {
+    fftSize: number,
+    bufferLen: number,
+    smoothingTimeConstant: number,
+    minCaptureFreq: number,         // in Hz
+    maxCaptureFreq: number,        // in Hz
+    noiseCaptureDuration: number, // in ms
+    minNoiseLevel: number,         // from 0 to 1
+    maxNoiseLevel: number,         // from 0 to 1
+    avgNoiseMultiplier: number,     // from 0 to 1
+    onVoiceStart?: () => void;
+    onVoiceStop?: () => void;
+    onUpdate?: (value: number) => void;
+  };
+
+  export default function vad(
+    audioContext: AudioContext,
+    stream: MediaStream,
+    options: VADOptions,
+  ): void;
+}

package/dist/custom/speech_recognition_frontend/voiceActivityDetection.ts

@@ -0,0 +1,151 @@
+import vad from 'voice-activity-detection';
+
+let currentStream: MediaStream | null = null;
+let vadInstance: any = null;
+let audioContext: AudioContext | null = null;
+let mediaRecorder: MediaRecorder | null = null;
+let recordedChunks: BlobPart[] = [];
+let wasVoiceStarted = false;
+
+export const CALIBRATION_DURATION = 1000; // in ms
+
+export async function requestMicAndStartVAD(
+  onVoiceStopCallback: () => void,
+  onVoiceStartCallback: () => void,
+  onUpdateCallback: (amplitude: number) => void
+) {
+  return new Promise<void>((resolve, reject) => {
+    try {
+      audioContext = new AudioContext();
+
+      navigator.mediaDevices
+        .getUserMedia({ audio: true })
+        .then((stream) => {
+          currentStream = stream;
+          startRecording(stream);
+          resolve();
+          startUserMedia(audioContext as AudioContext, stream, onVoiceStartCallback, onVoiceStopCallback, onUpdateCallback);
+        })
+        .catch((error) => {
+          handleMicConnectError();
+          reject(error);
+        });
+    } catch (e) {
+      handleUserMediaError();
+      reject(e);
+    }
+  });
+}
+
+function handleUserMediaError() {
+  console.error('Mic input is not supported by the browser.');
+}
+
+function handleMicConnectError() {
+  console.error('Could not connect microphone. Possible rejected by the user or is blocked by the browser.');
+}
+
+export async function stopUserMedia() {
+  wasVoiceStarted = false;
+  if (vadInstance && vadInstance.destroy) {
+    vadInstance.destroy();
+    vadInstance = null;
+  }
+
+  if (currentStream) {
+    currentStream.getTracks().forEach(track => track.stop());
+    currentStream = null;
+  }
+
+  if (audioContext) {
+    audioContext.close();
+    audioContext = null;
+  }
+
+  if (mediaRecorder) {
+    mediaRecorder.stop();
+    mediaRecorder = null;
+  }
+
+}
+
+function startRecording(stream: MediaStream) {
+  recordedChunks = [];
+  mediaRecorder = new MediaRecorder(stream);
+  mediaRecorder.ondataavailable = (event: BlobEvent) => {
+    if (event.data.size > 0) {
+      recordedChunks.push(event.data);
+    }
+  };
+  mediaRecorder.start();
+}
+
+export async function getRecorder(): Promise<Blob | null> {
+  if (!mediaRecorder) {
+    return Promise.resolve(null);
+  }
+
+  const recorder = mediaRecorder;
+  mediaRecorder = null;
+
+  const finalizeBlob = () => {
+    const blob = new Blob(recordedChunks, { type: recorder.mimeType || 'audio/webm' });
+    recordedChunks = [];
+    return blob;
+  };
+
+  if (recorder.state === 'inactive') {
+    return Promise.resolve(finalizeBlob());
+  }
+
+  return new Promise<Blob>((resolve, reject) => {
+    recorder.onstop = () => {
+      resolve(finalizeBlob());
+    };
+    recorder.onerror = () => {
+      recordedChunks = [];
+      reject(new Error('Failed to finalize audio recording.'));
+    };
+    recorder.stop();
+  });
+}
+
+function startUserMedia(
+  audioContext: AudioContext, 
+  stream: MediaStream, 
+  onVoiceStartCallback: () => void,
+  onVoiceStopCallback: () => void,
+  onUpdateCallback: (amplitude: number) => void
+) {
+  const options = {
+    fftSize: 1024,
+    bufferLen: 1024,
+    smoothingTimeConstant: 0.2,
+    minCaptureFreq: 85,         // in Hz
+    maxCaptureFreq: 255,        // in Hz
+    noiseCaptureDuration: CALIBRATION_DURATION, // in ms
+    minNoiseLevel: 0.5,         // from 0 to 1
+    maxNoiseLevel: 0.7,         // from 0 to 1
+    avgNoiseMultiplier: 1.2,
+    onVoiceStart() {
+      wasVoiceStarted = true;
+      if (!mediaRecorder || mediaRecorder.state === 'inactive') {
+        startRecording(currentStream as MediaStream);
+      }
+      console.log('👹👹👹voice start👹👹👹');
+      onVoiceStartCallback();
+    },
+    onVoiceStop() {
+      if (!wasVoiceStarted) {
+        return;
+      }
+      console.log('👿👿👿voice stop👿👿👿');
+      onVoiceStopCallback();
+    }, //Doesn't work properly, so we will handle it with onUpdate callback
+    onUpdate(val: number) {
+      onUpdateCallback(val);
+    }
+  };
+
+  vadInstance = vad(audioContext, stream, options);
+}

package/dist/custom/types.ts

@@ -25,7 +25,7 @@ export interface IToolGroup {
 }
 
 export interface IMessage {
-  role: 'user' | 'assistant';
+  role: 'user' | 'assistant' | 'system';
   metadata?: any,
   parts: IPart[];
 }
@@ -41,4 +41,54 @@ export interface ISessionsListItem {
   sessionId: string;
   title: string;
   timestamp: string;
-}
+}
+
+export type SpeechStreamEvent =
+  | {
+      type: 'error';
+      error: string;
+    }
+  | {
+      type: 'transcript';
+      data: {
+        text: string;
+        language?: string;
+      };
+    }
+  | {
+      type: 'speech-response';
+      data: {
+        transcript: {
+          text: string;
+          language?: string;
+        };
+        response: {
+          text: string;
+        };
+        sessionId: string;
+        turnId: string;
+      };
+    }
+  | {
+      type: 'audio-start';
+      data: {
+        mimeType: string;
+        format: string;
+      };
+    }
+  | {
+      type: 'audio-delta';
+      data: {
+        base64: string;
+      };
+    }
+  | {
+      type: 'audio-done';
+    }
+  | {
+      type: 'data-tool-call';
+      data: any;
+    }
+  | {
+      type: 'finish';
+    };