@adminforth/agent 1.37.0 → 1.38.0

package/custom/composables/agentStore/constants.ts

@@ -9,4 +9,11 @@ export const MIN_WIDTH = 25;
 export const DEFAULT_TEXTAREA_PLACEHOLDER = 'Type a message...';
 export const PLACEHOLDER_TYPING_DELAY_MS = 60;
 export const PLACEHOLDER_DELETING_DELAY_MS = 35;
-export const PLACEHOLDER_HOLD_DELAY_MS = 3000;
+export const PLACEHOLDER_HOLD_DELAY_MS = 3000;
+export const PRE_SESSION_ID = 'pre-session';
+
+export enum RESERVED_SYSTEM_MESSAGE_CONTENT {
+  START_AUDIO_CHAT = 'START_AUDIO_CHAT',
+  END_AUDIO_CHAT = 'END_AUDIO_CHAT',
+  AGENT_RESPONSE_ABORTED = 'AGENT_RESPONSE_ABORTED'
+}

package/custom/composables/agentStore/useAgentSessions.ts

@@ -2,6 +2,8 @@ import type { ComputedRef, Ref, ShallowRef } from 'vue';
 import { callAdminForthApi } from '@/utils';
 import type { Chat } from '../../chat';
 import type { IAgentSession, ISessionsListItem, IPart } from '../../types';
+import { PRE_SESSION_ID } from './constants';
+import { ChatStatus } from 'ai';
 import { useI18n } from 'vue-i18n';
 
 type AdminforthLike = {
@@ -91,8 +93,8 @@ export function createAgentSessionManager({
   }
 
   async function deletePreSession() {
-    sessionList.value = sessionList.value.filter((s: ISessionsListItem) => s.sessionId !== 'pre-session');
-    if (activeSessionId.value === 'pre-session') {
+    sessionList.value = sessionList.value.filter((s: ISessionsListItem) => s.sessionId !== PRE_SESSION_ID);
+    if (activeSessionId.value === PRE_SESSION_ID) {
       activeSessionId.value = null;
       currentSession.value = null;
     }
@@ -129,7 +131,7 @@ export function createAgentSessionManager({
     if (!message || isResponseInProgress.value) {
       return;
     }
-    if (!currentSession.value || currentSession.value.sessionId === 'pre-session') {
+    if (!currentSession.value || currentSession.value.sessionId === PRE_SESSION_ID) {
       await createNewSession(message);
     }
     currentSession.value!.timestamp = new Date().toISOString();
@@ -146,27 +148,27 @@ export function createAgentSessionManager({
 
   async function createPreSession() {
     saveCurrentSessionInCache();
-    if (!sessionList.value.some((s: ISessionsListItem) => s.sessionId === 'pre-session')) {
+    if (!sessionList.value.some((s: ISessionsListItem) => s.sessionId === PRE_SESSION_ID)) {
       sessionList.value.unshift({
-        sessionId: 'pre-session',
+        sessionId: PRE_SESSION_ID,
         title: 'New Session',
         timestamp: new Date().toISOString(),
       });
     }
 
-    activeSessionId.value = 'pre-session';
+    activeSessionId.value = PRE_SESSION_ID;
     currentSession.value = {
-      sessionId: 'pre-session',
+      sessionId: PRE_SESSION_ID,
       title: 'New Session',
       timestamp: new Date().toISOString(),
       messages: [],
     };
-    sessions.value['pre-session'] = currentSession.value;
-    setCurrentChat('pre-session');
+    sessions.value[PRE_SESSION_ID] = currentSession.value;
+    setCurrentChat(PRE_SESSION_ID);
   }
 
   async function deleteSession(sessionId: string) {
-    if (sessionId === 'pre-session') {
+    if (sessionId === PRE_SESSION_ID) {
       deletePreSession();
       return;
     }
@@ -236,7 +238,10 @@ export function createAgentSessionManager({
     currentChat.value?.messages.push(debugMessage);
   }
 
-  function addSystemMessage(message: string) {
+  async function addSystemMessage(message: string) {
+    if (!currentSession.value || currentSession.value.sessionId === PRE_SESSION_ID) {
+      await createNewSession('Audio chat');
+    }
     const systemMessage = {
       role: 'system',
       parts: [{
@@ -247,7 +252,7 @@ export function createAgentSessionManager({
     };
     currentChat.value?.messages.push(systemMessage);
     try {
-      const res = callAdminForthApi({
+      const res = await callAdminForthApi({
         method: 'POST',
         path: '/agent/add-system-message-to-turns',
         body: {
@@ -260,6 +265,69 @@ export function createAgentSessionManager({
     }
   }
 
+  function addAgentMessage(message: string) {
+    const agentMessage = {
+      role: 'assistant',
+      parts: [{
+        type: 'text',
+        text: message,
+        state: 'done',
+      }]
+    };
+    currentChat.value?.messages.push(agentMessage);
+  }
+
+  function updateLastAgentMessage(message: string) {
+    const lastMsg = currentChat.value?.lastMessage;
+    if (lastMsg && lastMsg.role === 'assistant') {
+      lastMsg.parts = [{
+        type: 'text',
+        text: message,
+        state: 'done',
+      }];
+      currentChat.value?.messages.splice(currentChat.value.messages.length - 1, 1, lastMsg);
+    } else {
+      addAgentMessage(message);
+    }
+  }
+
+  function addUserMessage(message: string) {
+    const userMessage = {
+      role: 'user',
+      parts: [{
+        type: 'text',
+        text: message,
+        state: 'done',
+      }]
+    };
+    currentChat.value?.messages.push(userMessage);
+  }
+
+
+  function addDataToolCallMessage(data: any) {
+    const lastMessage = currentChat.value?.lastMessage;
+    if (lastMessage.role === 'assistant') {
+      lastMessage.parts.push({
+        type: 'data-tool-call',
+        data,
+      });
+      currentChat.value?.messages.splice(currentChat.value.messages.length - 1, 1, lastMessage);
+    } else {
+      const toolCallMessage = {
+        role: 'assistant',
+        parts: [{
+          type: 'data-tool-call',
+          data,
+        }]
+      };
+      currentChat.value?.messages.push(toolCallMessage);
+    }
+  }
+
+  function setCurrentChatStatus(status: ChatStatus) {
+    (currentChat.value as any)?.setStatus({status});
+  }
+
   return {
     sendMessage,
     createPreSession,
@@ -268,5 +336,10 @@ export function createAgentSessionManager({
     deleteSession,
     addDebugMessage,
     addSystemMessage,
+    addAgentMessage,
+    addUserMessage,
+    addDataToolCallMessage,
+    setCurrentChatStatus,
+    updateLastAgentMessage
   };
 }

package/custom/composables/useAgentAudio.ts

@@ -0,0 +1,392 @@
+import adminforth from "@/adminforth";
+import { useAgentStore } from "./useAgentStore";
+import { defineStore } from 'pinia';
+import type { SpeechStreamEvent } from '../types';
+import { ref } from 'vue';
+import { getCurrentPageContext } from './agentStore/pageContext';
+
+type StreamingAudioState = {
+  mimeType: string;
+  mediaSource: MediaSource;
+  sourceBuffer: SourceBuffer | null;
+  pendingChunks: ArrayBuffer[];
+  hasStartedPlayback: boolean;
+  isDone: boolean;
+};
+
+const standByAudio = new Audio('./agentAudio/agent-processing.mp3');
+
+function playStandByAudio() {
+  standByAudio.currentTime = 0;
+  standByAudio.play()
+}
+
+function stopStandByAudio() {
+  standByAudio.pause();
+  standByAudio.currentTime = 0;
+}
+
+function restartStandByAudio() {
+  standByAudio.currentTime = 0;
+  playStandByAudio();
+}
+
+standByAudio.addEventListener('ended', () => {
+  if (!standByAudio.paused) {
+    restartStandByAudio();
+  }
+});
+
+export const useAgentAudio = defineStore('agentAudio', () => {
+  const agentStore = useAgentStore();
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
+  const isStreamingResponse = ref(false);
+    
+  let currentAbortController: AbortController | null = null;
+  let isPlaying = false;
+  let currentAudio: HTMLAudioElement | null = null;
+  let currentAudioObjectUrl: string | null = null;
+  let currentStreamingAudio: StreamingAudioState | null = null;
+  let bufferedAudioChunks: ArrayBuffer[] = [];
+  let bufferedAudioMimeType = 'audio/mpeg';
+
+  function stopGenerationAndAudio() {
+    agentAudioMode.value = null;
+    stopCurrentAudioPlayback();
+    currentAbortController?.abort();
+  }
+
+  async function sendAudioToServerAndHandleResponse(blob: Blob) {
+    currentAbortController = new AbortController();
+    const formData = new FormData();
+    formData.append('file', blob, 'user_prompt.webm');
+    formData.append('sessionId', agentStore.activeSessionId);
+    formData.append('mode', agentStore.activeModeName ?? '');
+    formData.append('timeZone', Intl.DateTimeFormat().resolvedOptions().timeZone);
+    formData.append('currentPage', JSON.stringify(getCurrentPageContext()));
+    const fullPath = `${import.meta.env.VITE_ADMINFORTH_PUBLIC_PATH || ''}/adminapi/v1/agent/speech-response`;
+    try {
+      agentAudioMode.value = 'transcribing';
+      const res = await fetch(fullPath, {
+        method: 'POST',
+        body: formData,
+        headers: {
+          Accept: 'text/event-stream',
+        },
+        signal: currentAbortController!.signal,
+      });
+      isStreamingResponse.value = true;
+      if (res.ok) {
+        agentAudioMode.value = 'streaming';
+        await readSpeechResponseStream(res);
+      } else {
+        console.error('Failed to transcribe audio:', res.statusText);
+        adminforth.alert({message: 'Failed to transcribe audio', variant: 'danger'});
+      }
+    } catch (error) {
+      if (error instanceof DOMException && error.name === 'AbortError') {
+        //
+      } else {
+        console.error('Error sending audio to server:', error);
+      }
+    } finally {
+      isStreamingResponse.value = false;
+      agentAudioMode.value = null;
+    }
+  }
+
+  async function readSpeechResponseStream(res: Response) {
+    const reader = res.body?.getReader();
+    if (!reader) {
+      adminforth.alert({message: 'Speech response stream is not available', variant: 'danger'});
+      return;
+    }
+
+    const decoder = new TextDecoder();
+    let buffer = '';
+
+    agentStore.setCurrentChatStatus('streaming');
+    try {
+      while (true) {
+        const { value, done } = await reader.read();
+
+        if (done) {
+          break;
+        }
+
+        buffer += decoder.decode(value, { stream: true });
+        const eventBlocks = buffer.split('\n\n');
+        buffer = eventBlocks.pop() ?? '';
+
+        for (const eventBlock of eventBlocks) {
+          await handleSpeechStreamEvent(eventBlock);
+        }
+      }
+
+      buffer += decoder.decode();
+
+      if (buffer.trim()) {
+        await handleSpeechStreamEvent(buffer);
+      }
+
+      finishAudioStream();
+    } finally {
+      reader.releaseLock();
+      agentStore.setCurrentChatStatus('ready');
+    }
+  }
+
+  async function handleSpeechStreamEvent(eventBlock: string) {
+    if (currentAbortController?.signal.aborted) {
+      return;
+    }
+    const data = eventBlock
+      .split('\n')
+      .filter((line) => line.startsWith('data:'))
+      .map((line) => line.slice(5).trimStart())
+      .join('\n');
+
+    if (!data || data === '[DONE]') {
+      return;
+    }
+
+    const event = JSON.parse(data) as SpeechStreamEvent;
+
+    if (event.type === 'error') {
+
+      return;
+    }
+
+    if (event.type === 'transcript') {
+      agentStore.addUserMessage(event.data.text);
+      agentStore.updateLastAgentMessage('');
+      return;
+    }
+
+    if (event.type === 'speech-response') {
+      // stopStandByAudio();
+      agentStore.setCurrentChatStatus('ready');
+      agentStore.addAgentMessage(event.data.response.text);
+      agentAudioMode.value = 'playingAgentResponse';
+      return;
+    }
+
+    if (event.type === 'audio-start') {
+      isStreamingResponse.value = false;
+      agentAudioMode.value = 'fetchingAudio';
+      initializeAudioStream(event.data.mimeType);
+      return;
+    }
+
+    if (event.type === 'audio-delta') {
+      appendAudioChunk(event.data.base64);
+      return;
+    }
+
+    if (event.type === 'audio-done') {
+      finishAudioStream();
+      return;
+    }
+
+    if (event.type === 'data-tool-call') {
+      // playStandByAudio();
+      agentStore.addDataToolCallMessage(event.data);
+    }
+  }
+
+  function setIsPlaying(value: boolean) {
+    isPlaying = value;
+
+    if (!currentAudio) {
+      return;
+    }
+
+    if (!isPlaying) {
+      currentAudio.pause();
+      currentAudio.currentTime = 0;
+      return;
+    }
+    agentAudioMode.value = 'playingAgentResponse';
+    void currentAudio.play().catch((error) => {
+      console.error('Failed to play audio:', error);
+    });
+  }
+
+  function initializeAudioStream(mimeType: string) {
+    stopCurrentAudioPlayback();
+    bufferedAudioMimeType = mimeType;
+
+    if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(mimeType)) {
+      return;
+    }
+
+    const mediaSource = new MediaSource();
+    currentAudioObjectUrl = URL.createObjectURL(mediaSource);
+    currentAudio = new Audio(currentAudioObjectUrl);
+    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
+    currentStreamingAudio = {
+      mimeType,
+      mediaSource,
+      sourceBuffer: null,
+      pendingChunks: [],
+      hasStartedPlayback: false,
+      isDone: false,
+    };
+
+    mediaSource.addEventListener('sourceopen', handleMediaSourceOpen, { once: true });
+  }
+
+  function handleMediaSourceOpen() {
+    if (!currentStreamingAudio) {
+      return;
+    }
+
+    try {
+      currentStreamingAudio.sourceBuffer = currentStreamingAudio.mediaSource.addSourceBuffer(currentStreamingAudio.mimeType);
+      currentStreamingAudio.sourceBuffer.mode = 'sequence';
+      currentStreamingAudio.sourceBuffer.addEventListener('updateend', flushStreamingAudioQueue);
+      flushStreamingAudioQueue();
+    } catch (error) {
+      console.error('Failed to initialize streaming audio playback:', error);
+      bufferedAudioChunks.push(...currentStreamingAudio.pendingChunks);
+      detachStreamingAudio();
+      destroyCurrentAudioElement();
+    }
+  }
+
+  function appendAudioChunk(base64: string) {
+    const chunk = base64ToArrayBuffer(base64);
+
+    if (!currentStreamingAudio) {
+      bufferedAudioChunks.push(chunk);
+      return;
+    }
+
+    currentStreamingAudio.pendingChunks.push(chunk);
+    flushStreamingAudioQueue();
+  }
+
+  function flushStreamingAudioQueue() {
+    if (!currentStreamingAudio?.sourceBuffer || currentStreamingAudio.sourceBuffer.updating) {
+      return;
+    }
+
+    const nextChunk = currentStreamingAudio.pendingChunks.shift();
+
+    if (nextChunk) {
+      currentStreamingAudio.sourceBuffer.appendBuffer(nextChunk);
+
+      if (!currentStreamingAudio.hasStartedPlayback) {
+        currentStreamingAudio.hasStartedPlayback = true;
+        setIsPlaying(true);
+      }
+
+      return;
+    }
+
+    if (currentStreamingAudio.isDone && currentStreamingAudio.mediaSource.readyState === 'open') {
+      currentStreamingAudio.mediaSource.endOfStream();
+    }
+  }
+
+  function finishAudioStream() {
+    if (currentStreamingAudio) {
+      currentStreamingAudio.isDone = true;
+      flushStreamingAudioQueue();
+      return;
+    }
+
+    if (!bufferedAudioChunks.length) {
+      return;
+    }
+
+    playAudioChunks(bufferedAudioChunks, bufferedAudioMimeType);
+    bufferedAudioChunks = [];
+  }
+
+  function detachStreamingAudio() {
+    if (currentStreamingAudio?.sourceBuffer) {
+      currentStreamingAudio.sourceBuffer.removeEventListener('updateend', flushStreamingAudioQueue);
+    }
+
+    currentStreamingAudio = null;
+  }
+
+  function destroyCurrentAudioElement() {
+    if (currentAudio) {
+      currentAudio.pause();
+      currentAudio.currentTime = 0;
+      currentAudio.src = '';
+      currentAudio.load();
+      currentAudio = null;
+    }
+
+    if (currentAudioObjectUrl) {
+      URL.revokeObjectURL(currentAudioObjectUrl);
+      currentAudioObjectUrl = null;
+    }
+
+    isPlaying = false;
+  }
+
+  function stopCurrentAudioPlayback(dontResetMode = false) {
+    bufferedAudioChunks = [];
+    bufferedAudioMimeType = 'audio/mpeg';
+    detachStreamingAudio();
+    destroyCurrentAudioElement();
+    if (!dontResetMode) {
+      agentAudioMode.value = null;
+    }
+  }
+
+  function handleAudioEnded() {
+    agentAudioMode.value = null;
+    stopCurrentAudioPlayback();
+  }
+
+  function playAudioChunks(chunks: ArrayBuffer[], mimeType: string) {
+    currentAudioObjectUrl = URL.createObjectURL(new Blob(chunks, { type: mimeType }));
+    currentAudio = new Audio(currentAudioObjectUrl);
+    currentAudio.addEventListener('ended', handleAudioEnded, { once: true });
+    setIsPlaying(true);
+  }
+
+  function base64ToArrayBuffer(base64: string) {
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+
+    for (let i = 0; i < binary.length; i += 1) {
+      bytes[i] = binary.charCodeAt(i);
+    }
+
+    return bytes.buffer;
+  }
+
+  function playBeep(freq = 800, duration = 0.05) {
+    const ctx = new AudioContext();
+    const osc = ctx.createOscillator();
+    const gain = ctx.createGain();
+
+    osc.frequency.value = freq;
+    osc.type = 'sine';
+
+    osc.connect(gain);
+    gain.connect(ctx.destination);
+
+    osc.start();
+
+    gain.gain.setValueAtTime(1, ctx.currentTime);
+    gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + duration);
+
+    osc.stop(ctx.currentTime + duration);
+  }
+
+  return {
+    sendAudioToServerAndHandleResponse,
+    stopGenerationAndAudio,
+    stopCurrentAudioPlayback,
+    playBeep,
+    agentAudioMode
+  };
+
+});

package/custom/composables/useAgentStore.ts

@@ -11,6 +11,8 @@ import {
   DEFAULT_CHAT_WIDTH,
   MAX_WIDTH,
   MIN_WIDTH,
+  RESERVED_SYSTEM_MESSAGE_CONTENT,
+  PRE_SESSION_ID
 } from './agentStore/constants';
 import { createAgentChatManager } from './agentStore/useAgentChat';
 import { createAgentPlaceholderController } from './agentStore/useAgentPlaceholder';
@@ -67,6 +69,36 @@ export const useAgentStore = defineStore('agent', () => {
     return window.localStorage.getItem(`${coreStore.config.brandName || 'adminforth'}-${key}`);
   }
 
+  const isAudioChatMode = ref(false);
+
+  const onBeforeChatCloseCallbacks: Array<() => Promise<void>> = [];
+  function registerOnBeforeChatCloseCallback(hook: () => Promise<void>) {
+    onBeforeChatCloseCallbacks.push(hook);
+  }
+
+  async function executeOnBeforeChatCloseCallbacks() {
+    for(const hook of onBeforeChatCloseCallbacks) {
+      try {
+        await hook();
+      } catch (error) {
+        console.error('Error executing onBeforeChatClose callback:', error);
+      }
+    }
+  }
+
+  function setIsAudioChatMode(isAudioChat: boolean) {
+    isAudioChatMode.value = isAudioChat;
+  }
+
+  watch(isAudioChatMode, (newVal: boolean) => {
+    if (newVal) {
+      addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.START_AUDIO_CHAT);
+    } else {
+      addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.END_AUDIO_CHAT);
+    }
+  });
+
+
   const isResponseInProgress = computed( () => {
     return currentChat.value?.status === 'streaming';
   });
@@ -79,6 +111,11 @@ export const useAgentStore = defineStore('agent', () => {
     deleteSession,
     addDebugMessage,
     addSystemMessage,
+    addAgentMessage,
+    addUserMessage,
+    addDataToolCallMessage,
+    setCurrentChatStatus,
+    updateLastAgentMessage
   } = createAgentSessionManager({
     activeSessionId,
     currentSession,
@@ -144,7 +181,7 @@ export const useAgentStore = defineStore('agent', () => {
       }
     }
     lastSessionId.value = getLocalStorageItem('lastSessionId');
-    if (lastSessionId.value && lastSessionId.value !== 'pre-session') {
+    if (lastSessionId.value && lastSessionId.value !== PRE_SESSION_ID) {
       setActiveSession(lastSessionId.value);
     }
     if (coreStore.isMobile) {
@@ -231,7 +268,8 @@ export const useAgentStore = defineStore('agent', () => {
     activeModeName.value = modeName;
   }
 
-  function closeChat() {
+  async function closeChat() {
+    await executeOnBeforeChatCloseCallbacks();
     if(isFullScreen.value) {
       document.body.style.overflow = '';
     }
@@ -273,7 +311,7 @@ export const useAgentStore = defineStore('agent', () => {
 
   function abortCurrentChatRequestAndAddSystemMessage() {
     abortCurrentChatRequest();
-    addSystemMessage('[Response generation aborted]');
+    addSystemMessage(RESERVED_SYSTEM_MESSAGE_CONTENT.AGENT_RESPONSE_ABORTED);
   }
 
   return {
@@ -315,9 +353,18 @@ export const useAgentStore = defineStore('agent', () => {
     DEFAULT_CHAT_WIDTH,
     MAX_WIDTH,
     MIN_WIDTH,
+    RESERVED_SYSTEM_MESSAGE_CONTENT,
     getLocalStorageItem,
     addDebugMessage,
     abortCurrentChatRequestAndAddSystemMessage,
-    addSystemMessage
+    addSystemMessage,
+    isAudioChatMode,
+    setIsAudioChatMode,
+    registerOnBeforeChatCloseCallback,
+    addAgentMessage,
+    addUserMessage,
+    addDataToolCallMessage,
+    setCurrentChatStatus,
+    updateLastAgentMessage
   }
-})
+})

package/custom/conversation_area/ConversationArea.vue

@@ -35,7 +35,7 @@
 
       <div 
         v-for="(message, index) in props.messages" :key="index"
-        class="flex flex-col w-full mt-2"
+        class="flex flex-col w-full mt-2 pb-2"
         :class="message.role === 'user' ? 'self-end' : 'self-start'"
         ref="messagesRefs"
       >