@adminforth/agent 1.41.2 → 1.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/build.log +2 -2
  2. package/custom/composables/useAgentAudio.ts +15 -6
  3. package/custom/speech_recognition_frontend/AudioLines.vue +1 -1
  4. package/custom/speech_recognition_frontend/MicrophoneButon.vue +43 -29
  5. package/dist/custom/composables/useAgentAudio.ts +15 -6
  6. package/dist/custom/speech_recognition_frontend/AudioLines.vue +1 -1
  7. package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue +43 -29
  8. package/package.json +1 -1
package/build.log CHANGED
@@ -58,5 +58,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
58
58
  custom/speech_recognition_frontend/types/
59
59
  custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
60
60
 
61
- sent 1,660,284 bytes received 860 bytes 3,322,288.00 bytes/sec
62
- total size is 1,656,368 speedup is 1.00
61
+ sent 1,661,051 bytes received 860 bytes 3,323,822.00 bytes/sec
62
+ total size is 1,657,135 speedup is 1.00
package/custom/composables/useAgentAudio.ts CHANGED
@@ -49,7 +49,7 @@ function restartStandByAudio() {
49
49
 
50
50
  export const useAgentAudio = defineStore('agentAudio', () => {
51
51
  const agentStore = useAgentStore();
52
- const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
52
+ const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
53
53
  const isStreamingResponse = ref(false);
54
54
 
55
55
  let currentAbortController: AbortController | null = null;
@@ -59,15 +59,21 @@ export const useAgentAudio = defineStore('agentAudio', () => {
59
59
  let currentStreamingAudio: StreamingAudioState | null = null;
60
60
  let bufferedAudioChunks: ArrayBuffer[] = [];
61
61
  let bufferedAudioMimeType = 'audio/mpeg';
62
+ let wasAudioResponseReceived = false;
62
63
 
63
64
  function stopGenerationAndAudio() {
64
- agentAudioMode.value = null;
65
+ setAudioModeReadyToRespond();
65
66
  stopCurrentAudioPlayback();
66
67
  currentAbortController?.abort();
67
68
  }
68
69
 
70
+ function setAudioModeReadyToRespond() {
71
+ agentAudioMode.value = 'readyToRespond';
72
+ }
73
+
69
74
  async function sendAudioToServerAndHandleResponse(blob: Blob) {
70
75
  currentAbortController = new AbortController();
76
+ wasAudioResponseReceived = false;
71
77
  const formData = new FormData();
72
78
  formData.append('file', blob, 'user_prompt.wav');
73
79
  formData.append('sessionId', agentStore.activeSessionId);
@@ -101,7 +107,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
101
107
  }
102
108
  } finally {
103
109
  isStreamingResponse.value = false;
104
- agentAudioMode.value = null;
110
+ if (!wasAudioResponseReceived) {
111
+ setAudioModeReadyToRespond();
112
+ }
105
113
  }
106
114
  }
107
115
 
@@ -177,14 +185,15 @@ export const useAgentAudio = defineStore('agentAudio', () => {
177
185
  stopStandByAudio();
178
186
  agentStore.setCurrentChatStatus('ready');
179
187
  agentStore.addAgentMessage(event.data.response.text);
180
- agentAudioMode.value = 'playingAgentResponse';
181
188
  return;
182
189
  }
183
190
 
184
191
  if (event.type === 'audio-start') {
192
+ wasAudioResponseReceived = true;
185
193
  isStreamingResponse.value = false;
186
194
  agentAudioMode.value = 'fetchingAudio';
187
195
  initializeAudioStream(event.data.mimeType);
196
+ agentAudioMode.value = 'playingAgentResponse';
188
197
  return;
189
198
  }
190
199
 
@@ -348,12 +357,12 @@ export const useAgentAudio = defineStore('agentAudio', () => {
348
357
  detachStreamingAudio();
349
358
  destroyCurrentAudioElement();
350
359
  if (!dontResetMode) {
351
- agentAudioMode.value = null;
360
+ setAudioModeReadyToRespond();
352
361
  }
353
362
  }
354
363
 
355
364
  function handleAudioEnded() {
356
- agentAudioMode.value = null;
365
+ setAudioModeReadyToRespond();
357
366
  stopCurrentAudioPlayback();
358
367
  }
359
368
 
package/custom/speech_recognition_frontend/AudioLines.vue CHANGED
@@ -5,7 +5,7 @@
5
5
  class="bg-white w-[0.2rem] rounded-sm transition-all duration-100 ease-out"
6
6
  :style="{ height }"
7
7
  />
8
- <p v-if="isRecording" class="text-white ml-2">End</p>
8
+ <p v-if="isRecording" class="text-white ml-2">{{ $t('End') }}</p>
9
9
  </template>
10
10
 
11
11
  <script setup lang="ts">
package/custom/speech_recognition_frontend/MicrophoneButon.vue CHANGED
@@ -1,27 +1,39 @@
1
1
  <template>
2
- <button
3
- class="absolute bottom-2 h-9 bg-lightPrimary dark:bg-darkPrimary
4
- hover:opacity-90 rounded-full flex items-center justify-center
5
- transition-all duration-300 ease-in-out overflow-hidden"
6
- :class="[isAudioChatMode ? 'w-32 px-2': 'w-9', !agentStore.isAudioChatMode ? 'right-16': 'right-1/2 translate-x-1/2']"
7
- @click="toggleChatMode"
8
- >
9
- <div class="w-5 h-5 flex items-center justify-center">
10
- <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
11
- <AudioLines
12
- :showAnimation="showAudioWavesAnimation"
13
- :isRecording="microphoneButtonMode === 'listen'"
14
- :amplitude="audioAmplitude"
15
- />
2
+ <div
3
+ class="absolute bottom-2 flex items-center justify-center z-10 gap-4"
4
+ :class="[!agentStore.isAudioChatMode ? 'right-16': 'right-1/2 translate-x-1/2']"
5
+ >
6
+ <button
7
+ v-if="isAudioChatMode && microphoneButtonMode === 'generating'"
8
+ class="bg-red-500 hover:bg-red-600 dark:bg-red-700 dark:hover:bg-red-800 h-9 flex items-center justify-center rounded-full gap-2 text-white px-4"
9
+ @click="stopCurrentGeneration()"
10
+ >
11
+ <div class="w-3 h-3 bg-white rounded-full"/>
12
+ {{ $t('Break') }}
13
+ </button>
14
+ <button
15
+ class="h-9 bg-lightPrimary dark:bg-darkPrimary
16
+ hover:opacity-90 rounded-full flex items-center justify-center
17
+ transition-all duration-300 ease-in-out overflow-hidden"
18
+ :class="[isAudioChatMode ? 'w-32 px-2': 'w-9']"
19
+ @click="toggleChatMode"
20
+ >
21
+ <div class="w-5 h-5 flex items-center justify-center">
22
+ <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
23
+ <AudioLines
24
+ :showAnimation="showAudioWavesAnimation"
25
+ :isRecording="microphoneButtonMode === 'listen'"
26
+ :amplitude="audioAmplitude"
27
+ />
28
+ </div>
29
+ <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
30
+ <span class="w-3 h-3 bg-white rounded-sm" />
31
+ {{ $t('End') }}
32
+ </div>
33
+ <Spinner v-else class="w-4 h-4 text-lightButtonsText dark:text-darkButtonsText fill-lightButtonsBackground dark:fill-darkPrimary" />
16
34
  </div>
17
- <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
18
- <span class="w-3 h-3 bg-white rounded-sm" />
19
- {{ $t('Stop') }}
20
- </div>
21
- <Spinner v-else class="w-4 h-4 text-lightButtonsText dark:text-darkButtonsText fill-lightButtonsBackground dark:fill-darkPrimary" />
22
- </div>
23
-
24
- </button>
35
+ </button>
36
+ </div>
25
37
  </template>
26
38
 
27
39
 
@@ -63,7 +75,7 @@ onMounted(() => {
63
75
  });
64
76
  });
65
77
 
66
- watch(agentAudioMode, (newVal) => {
78
+ watch(agentAudioMode, async (newVal) => {
67
79
  if(newVal === 'streaming') {
68
80
  stopCurrentAudioPlayback(true);
69
81
  microphoneButtonMode.value = 'generating';
@@ -72,16 +84,22 @@ watch(agentAudioMode, (newVal) => {
72
84
  } else if (newVal === 'fetchingAudio') {
73
85
  //Generation is done, waiting for audio to be ready
74
86
  } else if (newVal === 'playingAgentResponse') {
75
- // Audio is playing
76
- } else {
87
+ // Audio is playing'
88
+ } else if (newVal === 'readyToRespond') {
77
89
  if(isAudioChatMode.value) {
78
90
  microphoneButtonMode.value = 'listen';
91
+ await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
79
92
  } else {
80
93
  microphoneButtonMode.value = 'off';
81
94
  }
82
95
  }
83
96
  })
84
97
 
98
+ function stopCurrentGeneration() {
99
+ stopGenerationAndAudio();
100
+ agentAudio.playBeep(1000);
101
+ }
102
+
85
103
  function toggleChatMode() {
86
104
  agentStore.setIsAudioChatMode(!isAudioChatMode.value);
87
105
  if (isAudioChatMode.value) {
@@ -142,12 +160,8 @@ async function sendRecordForTranscription() {
142
160
  showAudioWavesAnimation.value = false;
143
161
  const recordBlob = await getRecord();
144
162
  if (recordBlob) {
145
- console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
146
163
  onStopRecording();
147
164
  await sendAudioToServerAndHandleResponse(recordBlob);
148
- if (agentStore.isAudioChatMode) {
149
- await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
150
- }
151
165
  } else {
152
166
  console.error('No audio recorded');
153
167
  }
package/dist/custom/composables/useAgentAudio.ts CHANGED
@@ -49,7 +49,7 @@ function restartStandByAudio() {
49
49
 
50
50
  export const useAgentAudio = defineStore('agentAudio', () => {
51
51
  const agentStore = useAgentStore();
52
- const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
52
+ const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
53
53
  const isStreamingResponse = ref(false);
54
54
 
55
55
  let currentAbortController: AbortController | null = null;
@@ -59,15 +59,21 @@ export const useAgentAudio = defineStore('agentAudio', () => {
59
59
  let currentStreamingAudio: StreamingAudioState | null = null;
60
60
  let bufferedAudioChunks: ArrayBuffer[] = [];
61
61
  let bufferedAudioMimeType = 'audio/mpeg';
62
+ let wasAudioResponseReceived = false;
62
63
 
63
64
  function stopGenerationAndAudio() {
64
- agentAudioMode.value = null;
65
+ setAudioModeReadyToRespond();
65
66
  stopCurrentAudioPlayback();
66
67
  currentAbortController?.abort();
67
68
  }
68
69
 
70
+ function setAudioModeReadyToRespond() {
71
+ agentAudioMode.value = 'readyToRespond';
72
+ }
73
+
69
74
  async function sendAudioToServerAndHandleResponse(blob: Blob) {
70
75
  currentAbortController = new AbortController();
76
+ wasAudioResponseReceived = false;
71
77
  const formData = new FormData();
72
78
  formData.append('file', blob, 'user_prompt.wav');
73
79
  formData.append('sessionId', agentStore.activeSessionId);
@@ -101,7 +107,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
101
107
  }
102
108
  } finally {
103
109
  isStreamingResponse.value = false;
104
- agentAudioMode.value = null;
110
+ if (!wasAudioResponseReceived) {
111
+ setAudioModeReadyToRespond();
112
+ }
105
113
  }
106
114
  }
107
115
 
@@ -177,14 +185,15 @@ export const useAgentAudio = defineStore('agentAudio', () => {
177
185
  stopStandByAudio();
178
186
  agentStore.setCurrentChatStatus('ready');
179
187
  agentStore.addAgentMessage(event.data.response.text);
180
- agentAudioMode.value = 'playingAgentResponse';
181
188
  return;
182
189
  }
183
190
 
184
191
  if (event.type === 'audio-start') {
192
+ wasAudioResponseReceived = true;
185
193
  isStreamingResponse.value = false;
186
194
  agentAudioMode.value = 'fetchingAudio';
187
195
  initializeAudioStream(event.data.mimeType);
196
+ agentAudioMode.value = 'playingAgentResponse';
188
197
  return;
189
198
  }
190
199
 
@@ -348,12 +357,12 @@ export const useAgentAudio = defineStore('agentAudio', () => {
348
357
  detachStreamingAudio();
349
358
  destroyCurrentAudioElement();
350
359
  if (!dontResetMode) {
351
- agentAudioMode.value = null;
360
+ setAudioModeReadyToRespond();
352
361
  }
353
362
  }
354
363
 
355
364
  function handleAudioEnded() {
356
- agentAudioMode.value = null;
365
+ setAudioModeReadyToRespond();
357
366
  stopCurrentAudioPlayback();
358
367
  }
359
368
 
package/dist/custom/speech_recognition_frontend/AudioLines.vue CHANGED
@@ -5,7 +5,7 @@
5
5
  class="bg-white w-[0.2rem] rounded-sm transition-all duration-100 ease-out"
6
6
  :style="{ height }"
7
7
  />
8
- <p v-if="isRecording" class="text-white ml-2">End</p>
8
+ <p v-if="isRecording" class="text-white ml-2">{{ $t('End') }}</p>
9
9
  </template>
10
10
 
11
11
  <script setup lang="ts">
package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue CHANGED
@@ -1,27 +1,39 @@
1
1
  <template>
2
- <button
3
- class="absolute bottom-2 h-9 bg-lightPrimary dark:bg-darkPrimary
4
- hover:opacity-90 rounded-full flex items-center justify-center
5
- transition-all duration-300 ease-in-out overflow-hidden"
6
- :class="[isAudioChatMode ? 'w-32 px-2': 'w-9', !agentStore.isAudioChatMode ? 'right-16': 'right-1/2 translate-x-1/2']"
7
- @click="toggleChatMode"
8
- >
9
- <div class="w-5 h-5 flex items-center justify-center">
10
- <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
11
- <AudioLines
12
- :showAnimation="showAudioWavesAnimation"
13
- :isRecording="microphoneButtonMode === 'listen'"
14
- :amplitude="audioAmplitude"
15
- />
2
+ <div
3
+ class="absolute bottom-2 flex items-center justify-center z-10 gap-4"
4
+ :class="[!agentStore.isAudioChatMode ? 'right-16': 'right-1/2 translate-x-1/2']"
5
+ >
6
+ <button
7
+ v-if="isAudioChatMode && microphoneButtonMode === 'generating'"
8
+ class="bg-red-500 hover:bg-red-600 dark:bg-red-700 dark:hover:bg-red-800 h-9 flex items-center justify-center rounded-full gap-2 text-white px-4"
9
+ @click="stopCurrentGeneration()"
10
+ >
11
+ <div class="w-3 h-3 bg-white rounded-full"/>
12
+ {{ $t('Break') }}
13
+ </button>
14
+ <button
15
+ class="h-9 bg-lightPrimary dark:bg-darkPrimary
16
+ hover:opacity-90 rounded-full flex items-center justify-center
17
+ transition-all duration-300 ease-in-out overflow-hidden"
18
+ :class="[isAudioChatMode ? 'w-32 px-2': 'w-9']"
19
+ @click="toggleChatMode"
20
+ >
21
+ <div class="w-5 h-5 flex items-center justify-center">
22
+ <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
23
+ <AudioLines
24
+ :showAnimation="showAudioWavesAnimation"
25
+ :isRecording="microphoneButtonMode === 'listen'"
26
+ :amplitude="audioAmplitude"
27
+ />
28
+ </div>
29
+ <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
30
+ <span class="w-3 h-3 bg-white rounded-sm" />
31
+ {{ $t('End') }}
32
+ </div>
33
+ <Spinner v-else class="w-4 h-4 text-lightButtonsText dark:text-darkButtonsText fill-lightButtonsBackground dark:fill-darkPrimary" />
16
34
  </div>
17
- <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
18
- <span class="w-3 h-3 bg-white rounded-sm" />
19
- {{ $t('Stop') }}
20
- </div>
21
- <Spinner v-else class="w-4 h-4 text-lightButtonsText dark:text-darkButtonsText fill-lightButtonsBackground dark:fill-darkPrimary" />
22
- </div>
23
-
24
- </button>
35
+ </button>
36
+ </div>
25
37
  </template>
26
38
 
27
39
 
@@ -63,7 +75,7 @@ onMounted(() => {
63
75
  });
64
76
  });
65
77
 
66
- watch(agentAudioMode, (newVal) => {
78
+ watch(agentAudioMode, async (newVal) => {
67
79
  if(newVal === 'streaming') {
68
80
  stopCurrentAudioPlayback(true);
69
81
  microphoneButtonMode.value = 'generating';
@@ -72,16 +84,22 @@ watch(agentAudioMode, (newVal) => {
72
84
  } else if (newVal === 'fetchingAudio') {
73
85
  //Generation is done, waiting for audio to be ready
74
86
  } else if (newVal === 'playingAgentResponse') {
75
- // Audio is playing
76
- } else {
87
+ // Audio is playing'
88
+ } else if (newVal === 'readyToRespond') {
77
89
  if(isAudioChatMode.value) {
78
90
  microphoneButtonMode.value = 'listen';
91
+ await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
79
92
  } else {
80
93
  microphoneButtonMode.value = 'off';
81
94
  }
82
95
  }
83
96
  })
84
97
 
98
+ function stopCurrentGeneration() {
99
+ stopGenerationAndAudio();
100
+ agentAudio.playBeep(1000);
101
+ }
102
+
85
103
  function toggleChatMode() {
86
104
  agentStore.setIsAudioChatMode(!isAudioChatMode.value);
87
105
  if (isAudioChatMode.value) {
@@ -142,12 +160,8 @@ async function sendRecordForTranscription() {
142
160
  showAudioWavesAnimation.value = false;
143
161
  const recordBlob = await getRecord();
144
162
  if (recordBlob) {
145
- console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
146
163
  onStopRecording();
147
164
  await sendAudioToServerAndHandleResponse(recordBlob);
148
- if (agentStore.isAudioChatMode) {
149
- await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
150
- }
151
165
  } else {
152
166
  console.error('No audio recorded');
153
167
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adminforth/agent",
3
- "version": "1.41.2",
3
+ "version": "1.42.0",
4
4
  "main": "dist/index.js",
5
5
  "types": "dist/index.d.ts",
6
6
  "type": "module",