@adminforth/agent 1.37.0 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/languageDetect.ts +0 -8
- package/agent/simpleAgent.ts +5 -5
- package/agent/systemPrompt.ts +35 -4
- package/agent/toolCallEvents.ts +31 -2
- package/agent/tools/apiTool.ts +1 -1
- package/agentResponseEvents.ts +197 -0
- package/apiBasedTools.ts +118 -284
- package/build.log +12 -2
- package/custom/ChatSurface.vue +31 -21
- package/custom/composables/agentAudio/agent-processing.mp3 +0 -0
- package/custom/composables/agentStore/constants.ts +8 -1
- package/custom/composables/agentStore/useAgentSessions.ts +85 -12
- package/custom/composables/useAgentAudio.ts +392 -0
- package/custom/composables/useAgentStore.ts +52 -5
- package/custom/conversation_area/ConversationArea.vue +1 -1
- package/custom/conversation_area/MessageRenderer.vue +12 -1
- package/custom/conversation_area/SystemMessageRenderer.vue +28 -0
- package/custom/conversation_area/TextRenderer.vue +4 -3
- package/custom/conversation_area/ToolRenderer.vue +1 -1
- package/custom/package.json +2 -1
- package/custom/pnpm-lock.yaml +29 -0
- package/custom/speech_recognition_frontend/AudioLines.vue +97 -0
- package/custom/speech_recognition_frontend/MicrophoneButon.vue +157 -0
- package/custom/speech_recognition_frontend/types/voice-activity-detection.d.ts +22 -0
- package/custom/speech_recognition_frontend/voiceActivityDetection.ts +151 -0
- package/custom/types.ts +52 -2
- package/dist/agent/languageDetect.js +0 -6
- package/dist/agent/simpleAgent.js +4 -3
- package/dist/agent/systemPrompt.js +24 -3
- package/dist/agent/toolCallEvents.js +24 -2
- package/dist/agent/tools/apiTool.js +1 -1
- package/dist/agentResponseEvents.js +141 -0
- package/dist/apiBasedTools.js +95 -211
- package/dist/custom/ChatSurface.vue +31 -21
- package/dist/custom/composables/agentAudio/agent-processing.mp3 +0 -0
- package/dist/custom/composables/agentStore/constants.ts +8 -1
- package/dist/custom/composables/agentStore/useAgentSessions.ts +85 -12
- package/dist/custom/composables/useAgentAudio.ts +392 -0
- package/dist/custom/composables/useAgentStore.ts +52 -5
- package/dist/custom/conversation_area/ConversationArea.vue +1 -1
- package/dist/custom/conversation_area/MessageRenderer.vue +12 -1
- package/dist/custom/conversation_area/SystemMessageRenderer.vue +28 -0
- package/dist/custom/conversation_area/TextRenderer.vue +4 -3
- package/dist/custom/conversation_area/ToolRenderer.vue +1 -1
- package/dist/custom/package.json +2 -1
- package/dist/custom/pnpm-lock.yaml +29 -0
- package/dist/custom/speech_recognition_frontend/AudioLines.vue +97 -0
- package/dist/custom/speech_recognition_frontend/MicrophoneButon.vue +157 -0
- package/dist/custom/speech_recognition_frontend/types/voice-activity-detection.d.ts +22 -0
- package/dist/custom/speech_recognition_frontend/voiceActivityDetection.ts +151 -0
- package/dist/custom/types.ts +52 -2
- package/dist/index.js +290 -400
- package/index.ts +318 -492
- package/package.json +3 -2
- package/types.ts +1 -1
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
import adminforth from "@/adminforth";
|
|
2
|
+
import { useAgentStore } from "./useAgentStore";
|
|
3
|
+
import { defineStore } from 'pinia';
|
|
4
|
+
import type { SpeechStreamEvent } from '../types';
|
|
5
|
+
import { ref } from 'vue';
|
|
6
|
+
import { getCurrentPageContext } from './agentStore/pageContext';
|
|
7
|
+
|
|
8
|
+
type StreamingAudioState = {
|
|
9
|
+
mimeType: string;
|
|
10
|
+
mediaSource: MediaSource;
|
|
11
|
+
sourceBuffer: SourceBuffer | null;
|
|
12
|
+
pendingChunks: ArrayBuffer[];
|
|
13
|
+
hasStartedPlayback: boolean;
|
|
14
|
+
isDone: boolean;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const standByAudio = new Audio('./agentAudio/agent-processing.mp3');
|
|
18
|
+
|
|
19
|
+
function playStandByAudio() {
|
|
20
|
+
standByAudio.currentTime = 0;
|
|
21
|
+
standByAudio.play()
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function stopStandByAudio() {
|
|
25
|
+
standByAudio.pause();
|
|
26
|
+
standByAudio.currentTime = 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function restartStandByAudio() {
|
|
30
|
+
standByAudio.currentTime = 0;
|
|
31
|
+
playStandByAudio();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
standByAudio.addEventListener('ended', () => {
|
|
35
|
+
if (!standByAudio.paused) {
|
|
36
|
+
restartStandByAudio();
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export const useAgentAudio = defineStore('agentAudio', () => {
|
|
41
|
+
const agentStore = useAgentStore();
|
|
42
|
+
const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
|
|
43
|
+
const isStreamingResponse = ref(false);
|
|
44
|
+
|
|
45
|
+
let currentAbortController: AbortController | null = null;
|
|
46
|
+
let isPlaying = false;
|
|
47
|
+
let currentAudio: HTMLAudioElement | null = null;
|
|
48
|
+
let currentAudioObjectUrl: string | null = null;
|
|
49
|
+
let currentStreamingAudio: StreamingAudioState | null = null;
|
|
50
|
+
let bufferedAudioChunks: ArrayBuffer[] = [];
|
|
51
|
+
let bufferedAudioMimeType = 'audio/mpeg';
|
|
52
|
+
|
|
53
|
+
function stopGenerationAndAudio() {
|
|
54
|
+
agentAudioMode.value = null;
|
|
55
|
+
stopCurrentAudioPlayback();
|
|
56
|
+
currentAbortController?.abort();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function sendAudioToServerAndHandleResponse(blob: Blob) {
|
|
60
|
+
currentAbortController = new AbortController();
|
|
61
|
+
const formData = new FormData();
|
|
62
|
+
formData.append('file', blob, 'user_prompt.webm');
|
|
63
|
+
formData.append('sessionId', agentStore.activeSessionId);
|
|
64
|
+
formData.append('mode', agentStore.activeModeName ?? '');
|
|
65
|
+
formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);
|
|
66
|
+
formData.append('currentPage', JSON.stringify(getCurrentPageContext()));
|
|
67
|
+
const fullPath = `${import.meta.env.VITE_ADMINFORTH_PUBLIC_PATH || ''}/adminapi/v1/agent/speech-response`;
|
|
68
|
+
try {
|
|
69
|
+
agentAudioMode.value = 'transcribing';
|
|
70
|
+
const res = await fetch(fullPath, {
|
|
71
|
+
method: 'POST',
|
|
72
|
+
body: formData,
|
|
73
|
+
headers: {
|
|
74
|
+
Accept: 'text/event-stream',
|
|
75
|
+
},
|
|
76
|
+
signal: currentAbortController!.signal,
|
|
77
|
+
});
|
|
78
|
+
isStreamingResponse.value = true;
|
|
79
|
+
if (res.ok) {
|
|
80
|
+
agentAudioMode.value = 'streaming';
|
|
81
|
+
await readSpeechResponseStream(res);
|
|
82
|
+
} else {
|
|
83
|
+
console.error('Failed to transcribe audio:', res.statusText);
|
|
84
|
+
adminforth.alert({message: 'Failed to transcribe audio', variant: 'danger'});
|
|
85
|
+
}
|
|
86
|
+
} catch (error) {
|
|
87
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
88
|
+
//
|
|
89
|
+
} else {
|
|
90
|
+
console.error('Error sending audio to server:', error);
|
|
91
|
+
}
|
|
92
|
+
} finally {
|
|
93
|
+
isStreamingResponse.value = false;
|
|
94
|
+
agentAudioMode.value = null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function readSpeechResponseStream(res: Response) {
|
|
99
|
+
const reader = res.body?.getReader();
|
|
100
|
+
if (!reader) {
|
|
101
|
+
adminforth.alert({message: 'Speech response stream is not available', variant: 'danger'});
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const decoder = new TextDecoder();
|
|
106
|
+
let buffer = '';
|
|
107
|
+
|
|
108
|
+
agentStore.setCurrentChatStatus('streaming');
|
|
109
|
+
try {
|
|
110
|
+
while (true) {
|
|
111
|
+
const { value, done } = await reader.read();
|
|
112
|
+
|
|
113
|
+
if (done) {
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
buffer += decoder.decode(value, { stream: true });
|
|
118
|
+
const eventBlocks = buffer.split('\n\n');
|
|
119
|
+
buffer = eventBlocks.pop() ?? '';
|
|
120
|
+
|
|
121
|
+
for (const eventBlock of eventBlocks) {
|
|
122
|
+
await handleSpeechStreamEvent(eventBlock);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
buffer += decoder.decode();
|
|
127
|
+
|
|
128
|
+
if (buffer.trim()) {
|
|
129
|
+
await handleSpeechStreamEvent(buffer);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
finishAudioStream();
|
|
133
|
+
} finally {
|
|
134
|
+
reader.releaseLock();
|
|
135
|
+
agentStore.setCurrentChatStatus('ready');
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function handleSpeechStreamEvent(eventBlock: string) {
|
|
140
|
+
if (currentAbortController?.signal.aborted) {
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
const data = eventBlock
|
|
144
|
+
.split('\n')
|
|
145
|
+
.filter((line) => line.startsWith('data:'))
|
|
146
|
+
.map((line) => line.slice(5).trimStart())
|
|
147
|
+
.join('\n');
|
|
148
|
+
|
|
149
|
+
if (!data || data === '[DONE]') {
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const event = JSON.parse(data) as SpeechStreamEvent;
|
|
154
|
+
|
|
155
|
+
if (event.type === 'error') {
|
|
156
|
+
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (event.type === 'transcript') {
|
|
161
|
+
agentStore.addUserMessage(event.data.text);
|
|
162
|
+
agentStore.updateLastAgentMessage('');
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (event.type === 'speech-response') {
|
|
167
|
+
// stopStandByAudio();
|
|
168
|
+
agentStore.setCurrentChatStatus('ready');
|
|
169
|
+
agentStore.addAgentMessage(event.data.response.text);
|
|
170
|
+
agentAudioMode.value = 'playingAgentResponse';
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (event.type === 'audio-start') {
|
|
175
|
+
isStreamingResponse.value = false;
|
|
176
|
+
agentAudioMode.value = 'fetchingAudio';
|
|
177
|
+
initializeAudioStream(event.data.mimeType);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (event.type === 'audio-delta') {
|
|
182
|
+
appendAudioChunk(event.data.base64);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (event.type === 'audio-done') {
|
|
187
|
+
finishAudioStream();
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (event.type === 'data-tool-call') {
|
|
192
|
+
// playStandByAudio();
|
|
193
|
+
agentStore.addDataToolCallMessage(event.data);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function setIsPlaying(value: boolean) {
|
|
198
|
+
isPlaying = value;
|
|
199
|
+
|
|
200
|
+
if (!currentAudio) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (!isPlaying) {
|
|
205
|
+
currentAudio.pause();
|
|
206
|
+
currentAudio.currentTime = 0;
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
agentAudioMode.value = 'playingAgentResponse';
|
|
210
|
+
void currentAudio.play().catch((error) => {
|
|
211
|
+
console.error('Failed to play audio:', error);
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function initializeAudioStream(mimeType: string) {
|
|
216
|
+
stopCurrentAudioPlayback();
|
|
217
|
+
bufferedAudioMimeType = mimeType;
|
|
218
|
+
|
|
219
|
+
if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(mimeType)) {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const mediaSource = new MediaSource();
|
|
224
|
+
currentAudioObjectUrl = URL.createObjectURL(mediaSource);
|
|
225
|
+
currentAudio = new Audio(currentAudioObjectUrl);
|
|
226
|
+
currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
|
|
227
|
+
currentStreamingAudio = {
|
|
228
|
+
mimeType,
|
|
229
|
+
mediaSource,
|
|
230
|
+
sourceBuffer: null,
|
|
231
|
+
pendingChunks: [],
|
|
232
|
+
hasStartedPlayback: false,
|
|
233
|
+
isDone: false,
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
mediaSource.addEventListener('sourceopen', handleMediaSourceOpen, { once: true });
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function handleMediaSourceOpen() {
|
|
240
|
+
if (!currentStreamingAudio) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
try {
|
|
245
|
+
currentStreamingAudio.sourceBuffer = currentStreamingAudio.mediaSource.addSourceBuffer(currentStreamingAudio.mimeType);
|
|
246
|
+
currentStreamingAudio.sourceBuffer.mode = 'sequence';
|
|
247
|
+
currentStreamingAudio.sourceBuffer.addEventListener('updateend', flushStreamingAudioQueue);
|
|
248
|
+
flushStreamingAudioQueue();
|
|
249
|
+
} catch (error) {
|
|
250
|
+
console.error('Failed to initialize streaming audio playback:', error);
|
|
251
|
+
bufferedAudioChunks.push(...currentStreamingAudio.pendingChunks);
|
|
252
|
+
detachStreamingAudio();
|
|
253
|
+
destroyCurrentAudioElement();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function appendAudioChunk(base64: string) {
|
|
258
|
+
const chunk = base64ToArrayBuffer(base64);
|
|
259
|
+
|
|
260
|
+
if (!currentStreamingAudio) {
|
|
261
|
+
bufferedAudioChunks.push(chunk);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
currentStreamingAudio.pendingChunks.push(chunk);
|
|
266
|
+
flushStreamingAudioQueue();
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function flushStreamingAudioQueue() {
|
|
270
|
+
if (!currentStreamingAudio?.sourceBuffer || currentStreamingAudio.sourceBuffer.updating) {
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const nextChunk = currentStreamingAudio.pendingChunks.shift();
|
|
275
|
+
|
|
276
|
+
if (nextChunk) {
|
|
277
|
+
currentStreamingAudio.sourceBuffer.appendBuffer(nextChunk);
|
|
278
|
+
|
|
279
|
+
if (!currentStreamingAudio.hasStartedPlayback) {
|
|
280
|
+
currentStreamingAudio.hasStartedPlayback = true;
|
|
281
|
+
setIsPlaying(true);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (currentStreamingAudio.isDone && currentStreamingAudio.mediaSource.readyState === 'open') {
|
|
288
|
+
currentStreamingAudio.mediaSource.endOfStream();
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function finishAudioStream() {
|
|
293
|
+
if (currentStreamingAudio) {
|
|
294
|
+
currentStreamingAudio.isDone = true;
|
|
295
|
+
flushStreamingAudioQueue();
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (!bufferedAudioChunks.length) {
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
playAudioChunks(bufferedAudioChunks, bufferedAudioMimeType);
|
|
304
|
+
bufferedAudioChunks = [];
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function detachStreamingAudio() {
|
|
308
|
+
if (currentStreamingAudio?.sourceBuffer) {
|
|
309
|
+
currentStreamingAudio.sourceBuffer.removeEventListener('updateend', flushStreamingAudioQueue);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
currentStreamingAudio = null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function destroyCurrentAudioElement() {
|
|
316
|
+
if (currentAudio) {
|
|
317
|
+
currentAudio.pause();
|
|
318
|
+
currentAudio.currentTime = 0;
|
|
319
|
+
currentAudio.src = '';
|
|
320
|
+
currentAudio.load();
|
|
321
|
+
currentAudio = null;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
if (currentAudioObjectUrl) {
|
|
325
|
+
URL.revokeObjectURL(currentAudioObjectUrl);
|
|
326
|
+
currentAudioObjectUrl = null;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
isPlaying = false;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function stopCurrentAudioPlayback(dontResetMode = false) {
|
|
333
|
+
bufferedAudioChunks = [];
|
|
334
|
+
bufferedAudioMimeType = 'audio/mpeg';
|
|
335
|
+
detachStreamingAudio();
|
|
336
|
+
destroyCurrentAudioElement();
|
|
337
|
+
if (!dontResetMode) {
|
|
338
|
+
agentAudioMode.value = null;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function handleAudioEnded() {
|
|
343
|
+
agentAudioMode.value = null;
|
|
344
|
+
stopCurrentAudioPlayback();
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
|
|
348
|
+
currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
|
|
349
|
+
currentAudio = new Audio(currentAudioObjectUrl);
|
|
350
|
+
currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
|
|
351
|
+
setIsPlaying(true);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function base64ToArrayBuffer(base64: string) {
|
|
355
|
+
const binary = atob(base64);
|
|
356
|
+
const bytes = new Uint8Array(binary.length);
|
|
357
|
+
|
|
358
|
+
for (let i = 0; i < binary.length; i += 1) {
|
|
359
|
+
bytes[i] = binary.charCodeAt(i);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return bytes.buffer;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function playBeep(freq = 800, duration = 0.05) {
|
|
366
|
+
const ctx = new AudioContext();
|
|
367
|
+
const osc = ctx.createOscillator();
|
|
368
|
+
const gain = ctx.createGain();
|
|
369
|
+
|
|
370
|
+
osc.frequency.value = freq;
|
|
371
|
+
osc.type = 'sine';
|
|
372
|
+
|
|
373
|
+
osc.connect(gain);
|
|
374
|
+
gain.connect(ctx.destination);
|
|
375
|
+
|
|
376
|
+
osc.start();
|
|
377
|
+
|
|
378
|
+
gain.gain.setValueAtTime(1, ctx.currentTime);
|
|
379
|
+
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + duration);
|
|
380
|
+
|
|
381
|
+
osc.stop(ctx.currentTime + duration);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return {
|
|
385
|
+
sendAudioToServerAndHandleResponse,
|
|
386
|
+
stopGenerationAndAudio,
|
|
387
|
+
stopCurrentAudioPlayback,
|
|
388
|
+
playBeep,
|
|
389
|
+
agentAudioMode
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
});
|
|
@@ -11,6 +11,8 @@ import {
|
|
|
11
11
|
DEFAULT_CHAT_WIDTH,
|
|
12
12
|
MAX_WIDTH,
|
|
13
13
|
MIN_WIDTH,
|
|
14
|
+
RESERVED_SYSTEM_MESSAGE_CONTENT,
|
|
15
|
+
PRE_SESSION_ID
|
|
14
16
|
} from './agentStore/constants';
|
|
15
17
|
import { createAgentChatManager } from './agentStore/useAgentChat';
|
|
16
18
|
import { createAgentPlaceholderController } from './agentStore/useAgentPlaceholder';
|
|
@@ -67,6 +69,36 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
67
69
|
return window.localStorage.getItem(`${coreStore.config.brandName || 'adminforth'}-${key}`);
|
|
68
70
|
}
|
|
69
71
|
|
|
72
|
+
const isAudioChatMode = ref(false);
|
|
73
|
+
|
|
74
|
+
const onBeforeChatCloseCallbacks: Array<() => Promise<void>> = [];
|
|
75
|
+
function registerOnBeforeChatCloseCallback(hook: () => Promise<void>) {
|
|
76
|
+
onBeforeChatCloseCallbacks.push(hook);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function executeOnBeforeChatCloseCallbacks() {
|
|
80
|
+
for(const hook of onBeforeChatCloseCallbacks) {
|
|
81
|
+
try {
|
|
82
|
+
await hook();
|
|
83
|
+
} catch (error) {
|
|
84
|
+
console.error('Error executing onBeforeChatClose callback:', error);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function setIsAudioChatMode(isAudioChat: boolean) {
|
|
90
|
+
isAudioChatMode.value = isAudioChat;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
watch(isAudioChatMode, (newVal: boolean) => {
|
|
94
|
+
if (newVal) {
|
|
95
|
+
addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.START_AUDIO_CHAT);
|
|
96
|
+
} else {
|
|
97
|
+
addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.END_AUDIO_CHAT);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
|
|
70
102
|
const isResponseInProgress = computed( () => {
|
|
71
103
|
return currentChat.value?.status === 'streaming';
|
|
72
104
|
});
|
|
@@ -79,6 +111,11 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
79
111
|
deleteSession,
|
|
80
112
|
addDebugMessage,
|
|
81
113
|
addSystemMessage,
|
|
114
|
+
addAgentMessage,
|
|
115
|
+
addUserMessage,
|
|
116
|
+
addDataToolCallMessage,
|
|
117
|
+
setCurrentChatStatus,
|
|
118
|
+
updateLastAgentMessage
|
|
82
119
|
} = createAgentSessionManager({
|
|
83
120
|
activeSessionId,
|
|
84
121
|
currentSession,
|
|
@@ -144,7 +181,7 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
144
181
|
}
|
|
145
182
|
}
|
|
146
183
|
lastSessionId.value = getLocalStorageItem('lastSessionId');
|
|
147
|
-
if (lastSessionId.value && lastSessionId.value !==
|
|
184
|
+
if (lastSessionId.value && lastSessionId.value !== PRE_SESSION_ID) {
|
|
148
185
|
setActiveSession(lastSessionId.value);
|
|
149
186
|
}
|
|
150
187
|
if (coreStore.isMobile) {
|
|
@@ -231,7 +268,8 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
231
268
|
activeModeName.value = modeName;
|
|
232
269
|
}
|
|
233
270
|
|
|
234
|
-
function closeChat() {
|
|
271
|
+
async function closeChat() {
|
|
272
|
+
await executeOnBeforeChatCloseCallbacks();
|
|
235
273
|
if(isFullScreen.value) {
|
|
236
274
|
document.body.style.overflow = '';
|
|
237
275
|
}
|
|
@@ -273,7 +311,7 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
273
311
|
|
|
274
312
|
function abortCurrentChatRequestAndAddSystemMessage() {
|
|
275
313
|
abortCurrentChatRequest();
|
|
276
|
-
addSystemMessage(
|
|
314
|
+
addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.AGENT_RESPONSE_ABORTED);
|
|
277
315
|
}
|
|
278
316
|
|
|
279
317
|
return {
|
|
@@ -315,9 +353,18 @@ export const useAgentStore = defineStore('agent', () => {
|
|
|
315
353
|
DEFAULT_CHAT_WIDTH,
|
|
316
354
|
MAX_WIDTH,
|
|
317
355
|
MIN_WIDTH,
|
|
356
|
+
RESERVED_SYSTEM_MESSAGE_CONTENT,
|
|
318
357
|
getLocalStorageItem,
|
|
319
358
|
addDebugMessage,
|
|
320
359
|
abortCurrentChatRequestAndAddSystemMessage,
|
|
321
|
-
addSystemMessage
|
|
360
|
+
addSystemMessage,
|
|
361
|
+
isAudioChatMode,
|
|
362
|
+
setIsAudioChatMode,
|
|
363
|
+
registerOnBeforeChatCloseCallback,
|
|
364
|
+
addAgentMessage,
|
|
365
|
+
addUserMessage,
|
|
366
|
+
addDataToolCallMessage,
|
|
367
|
+
setCurrentChatStatus,
|
|
368
|
+
updateLastAgentMessage
|
|
322
369
|
}
|
|
323
|
-
})
|
|
370
|
+
})
|
|
@@ -8,11 +8,15 @@
|
|
|
8
8
|
:key="part.type"
|
|
9
9
|
>
|
|
10
10
|
<TextRenderer
|
|
11
|
-
v-if="part.type === 'text'"
|
|
11
|
+
v-if="part.type === 'text' && !checkIfMessageSystemMessage(part.text ?? '')"
|
|
12
12
|
:message="part.text"
|
|
13
13
|
:role="props.message.role"
|
|
14
14
|
:state="part.state ?? (props.message.role === 'user' ? 'done' : undefined)"
|
|
15
15
|
/>
|
|
16
|
+
<SystemMessageRenderer
|
|
17
|
+
v-else
|
|
18
|
+
:message="part.text"
|
|
19
|
+
/>
|
|
16
20
|
</template>
|
|
17
21
|
|
|
18
22
|
</template>
|
|
@@ -25,9 +29,16 @@
|
|
|
25
29
|
import type { IMessage } from '../types';
|
|
26
30
|
import { getMessageParts } from '../utils';
|
|
27
31
|
import ProcessingTimeline from './ProcessingTimeline.vue';
|
|
32
|
+
import SystemMessageRenderer from './SystemMessageRenderer.vue';
|
|
33
|
+
import { RESERVED_SYSTEM_MESSAGE_CONTENT } from '../composables/agentStore/constants';
|
|
28
34
|
|
|
29
35
|
const props = defineProps<{
|
|
30
36
|
message: IMessage
|
|
31
37
|
isLastMessageInChat: boolean
|
|
32
38
|
}>();
|
|
39
|
+
|
|
40
|
+
function checkIfMessageSystemMessage(message: IMessage): boolean {
|
|
41
|
+
const isReserved = Object.values(RESERVED_SYSTEM_MESSAGE_CONTENT).includes(message as RESERVED_SYSTEM_MESSAGE_CONTENT);
|
|
42
|
+
return isReserved;
|
|
43
|
+
}
|
|
33
44
|
</script>
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<template>
|
|
2
|
+
<p :class="`${TEXT_CLASSES}`" v-if="message === RESERVED_SYSTEM_MESSAGE_CONTENT.START_AUDIO_CHAT">
|
|
3
|
+
<IconPhoneSolid class="inline-block w-4 h-4 mr-1" />
|
|
4
|
+
{{$t('Audio chat started')}}
|
|
5
|
+
</p>
|
|
6
|
+
<p :class="`${TEXT_CLASSES}`" v-else-if="message === RESERVED_SYSTEM_MESSAGE_CONTENT.END_AUDIO_CHAT">
|
|
7
|
+
<IconPhoneHangupSolid class="inline-block w-4 h-4 mr-1" />
|
|
8
|
+
{{$t('Audio chat ended')}}
|
|
9
|
+
</p>
|
|
10
|
+
<p :class="`${TEXT_CLASSES}`" v-else-if="message === RESERVED_SYSTEM_MESSAGE_CONTENT.AGENT_RESPONSE_ABORTED">
|
|
11
|
+
{{$t('Agent response aborted')}}
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
</template>
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
<script setup lang="ts">
|
|
19
|
+
import { RESERVED_SYSTEM_MESSAGE_CONTENT } from '../composables/agentStore/constants';
|
|
20
|
+
import { IconPhoneSolid, IconPhoneHangupSolid } from '@iconify-prerendered/vue-flowbite';
|
|
21
|
+
|
|
22
|
+
const TEXT_CLASSES = 'text-center italic text-lightListTableHeadingText dark:text-darkListTableHeadingText flex items-center justify-center';
|
|
23
|
+
|
|
24
|
+
const props = defineProps<{
|
|
25
|
+
message: string
|
|
26
|
+
}>();
|
|
27
|
+
|
|
28
|
+
</script>
|
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
:components="incremarkComponents"
|
|
17
17
|
:incremark-options="incremarkOptions"
|
|
18
18
|
/>
|
|
19
|
-
<p v-else class="text-red-500 py-2">
|
|
19
|
+
<!-- <p v-else class="text-red-500 py-2">
|
|
20
20
|
{{ $t('No content to render') }}
|
|
21
|
-
</p>
|
|
21
|
+
</p> -->
|
|
22
22
|
</div>
|
|
23
23
|
</template>
|
|
24
24
|
|
|
@@ -28,11 +28,12 @@
|
|
|
28
28
|
import { useRouter } from 'vue-router';
|
|
29
29
|
import { useAgentStore } from '../composables/useAgentStore';
|
|
30
30
|
import { useCoreStore } from '@/stores/core';
|
|
31
|
+
import type { IMessage } from '../types';
|
|
31
32
|
|
|
32
33
|
const props = defineProps<{
|
|
33
34
|
message: string | undefined,
|
|
34
35
|
state: string | undefined,
|
|
35
|
-
role: '
|
|
36
|
+
role: IMessage['role']
|
|
36
37
|
}>();
|
|
37
38
|
|
|
38
39
|
const emit = defineEmits(['toggle-thoughts']);
|
|
@@ -109,7 +109,7 @@
|
|
|
109
109
|
const hasToolSections = computed(() => toolSections.value.length > 0);
|
|
110
110
|
|
|
111
111
|
onMounted(() => {
|
|
112
|
-
if (toolRendererRef.value) {
|
|
112
|
+
if (toolRendererRef.value && props.data.toolInfo) {
|
|
113
113
|
toolRendererInitialWidth.value = toolRendererRef.value.offsetWidth;
|
|
114
114
|
}
|
|
115
115
|
});
|
package/dist/custom/package.json
CHANGED
|
@@ -44,6 +44,9 @@ importers:
|
|
|
44
44
|
vega-embed:
|
|
45
45
|
specifier: ^7.1.0
|
|
46
46
|
version: 7.1.0(vega-lite@6.4.2(vega@6.2.0))(vega@6.2.0)
|
|
47
|
+
voice-activity-detection:
|
|
48
|
+
specifier: ^0.0.5
|
|
49
|
+
version: 0.0.5
|
|
47
50
|
|
|
48
51
|
packages:
|
|
49
52
|
|
|
@@ -263,6 +266,9 @@ packages:
|
|
|
263
266
|
peerDependencies:
|
|
264
267
|
zod: ^3.25.76 || ^4.1.8
|
|
265
268
|
|
|
269
|
+
analyser-frequency-average@1.0.0:
|
|
270
|
+
resolution: {integrity: sha512-Y8HRgDfMWpefR286IAT7w9WsZ2r2dLOAkUNz8SQgsTAM0GsM9SAAqr1psqOr1scN76cL0pfuNZoQTnuvdoM0RA==}
|
|
271
|
+
|
|
266
272
|
ansi-regex@6.2.2:
|
|
267
273
|
resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==}
|
|
268
274
|
engines: {node: '>=12'}
|
|
@@ -271,6 +277,9 @@ packages:
|
|
|
271
277
|
resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==}
|
|
272
278
|
engines: {node: '>=12'}
|
|
273
279
|
|
|
280
|
+
audio-frequency-to-index@2.0.0:
|
|
281
|
+
resolution: {integrity: sha512-7zqlDEAsEkPB0ORRhjBlsK7KBZQtdgLLQcmemFD2V2KHPH4flqzDOheWl+U69K0P/LA7J/H5YBNzNWaoS/7WAQ==}
|
|
282
|
+
|
|
274
283
|
ccount@2.0.1:
|
|
275
284
|
resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
|
|
276
285
|
|
|
@@ -286,6 +295,9 @@ packages:
|
|
|
286
295
|
character-reference-invalid@2.0.1:
|
|
287
296
|
resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==}
|
|
288
297
|
|
|
298
|
+
clamp@1.0.1:
|
|
299
|
+
resolution: {integrity: sha512-kgMuFyE78OC6Dyu3Dy7vcx4uy97EIbVxJB/B0eJ3bUNAkwdNcxYzgKltnyADiYwsR7SEqkkUPsEUT//OVS6XMA==}
|
|
300
|
+
|
|
289
301
|
cliui@9.0.1:
|
|
290
302
|
resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==}
|
|
291
303
|
engines: {node: '>=20'}
|
|
@@ -882,6 +894,9 @@ packages:
|
|
|
882
894
|
vfile@6.0.3:
|
|
883
895
|
resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==}
|
|
884
896
|
|
|
897
|
+
voice-activity-detection@0.0.5:
|
|
898
|
+
resolution: {integrity: sha512-Ezq+k0cICI67XR/R9KvTRhJ7/u6U0Zk9I+8VNs3hHTAvB6cA3glCa+cHgCRWSW/SV6cRrufSQsV1YxpNtsQb1g==}
|
|
899
|
+
|
|
885
900
|
vue@3.5.32:
|
|
886
901
|
resolution: {integrity: sha512-vM4z4Q9tTafVfMAK7IVzmxg34rSzTFMyIe0UUEijUCkn9+23lj0WRfA83dg7eQZIUlgOSGrkViIaCfqSAUXsMw==}
|
|
887
902
|
peerDependencies:
|
|
@@ -1187,10 +1202,18 @@ snapshots:
|
|
|
1187
1202
|
'@opentelemetry/api': 1.9.0
|
|
1188
1203
|
zod: 4.3.6
|
|
1189
1204
|
|
|
1205
|
+
analyser-frequency-average@1.0.0:
|
|
1206
|
+
dependencies:
|
|
1207
|
+
audio-frequency-to-index: 2.0.0
|
|
1208
|
+
|
|
1190
1209
|
ansi-regex@6.2.2: {}
|
|
1191
1210
|
|
|
1192
1211
|
ansi-styles@6.2.3: {}
|
|
1193
1212
|
|
|
1213
|
+
audio-frequency-to-index@2.0.0:
|
|
1214
|
+
dependencies:
|
|
1215
|
+
clamp: 1.0.1
|
|
1216
|
+
|
|
1194
1217
|
ccount@2.0.1: {}
|
|
1195
1218
|
|
|
1196
1219
|
character-entities-html4@2.1.0: {}
|
|
@@ -1201,6 +1224,8 @@ snapshots:
|
|
|
1201
1224
|
|
|
1202
1225
|
character-reference-invalid@2.0.1: {}
|
|
1203
1226
|
|
|
1227
|
+
clamp@1.0.1: {}
|
|
1228
|
+
|
|
1204
1229
|
cliui@9.0.1:
|
|
1205
1230
|
dependencies:
|
|
1206
1231
|
string-width: 7.2.0
|
|
@@ -2143,6 +2168,10 @@ snapshots:
|
|
|
2143
2168
|
'@types/unist': 3.0.3
|
|
2144
2169
|
vfile-message: 4.0.3
|
|
2145
2170
|
|
|
2171
|
+
voice-activity-detection@0.0.5:
|
|
2172
|
+
dependencies:
|
|
2173
|
+
analyser-frequency-average: 1.0.0
|
|
2174
|
+
|
|
2146
2175
|
vue@3.5.32:
|
|
2147
2176
|
dependencies:
|
|
2148
2177
|
'@vue/compiler-dom': 3.5.32
|