neoagent 2.3.1-beta.4 → 2.3.1-beta.41

package/server/services/voice/providers.js

@@ -6,6 +6,7 @@ const { AGENT_DATA_DIR } = require('../../../runtime/paths');
 const { getOpenAiClient } = require('./openaiClient');
 const { synthesizeSpeechBuffer } = require('./openaiSpeech');
 const { transcribeChunkWithDeepgram } = require('../recordings/deepgram');
+const { decryptLocalValue } = require('../../utils/local_secrets');
 
 const DEFAULT_STT_PROVIDER = 'openai';
 const DEFAULT_TTS_PROVIDER = 'openai';
@@ -36,6 +37,12 @@ const DEFAULT_GEMINI_TRANSCRIPTION_PROMPT =
   'Transcribe this audio verbatim. Return only the transcript text.';
 const EMOJI_SPEECH_REGEX =
   /[\p{Extended_Pictographic}\p{Emoji_Presentation}\p{Regional_Indicator}\u200D\uFE0F\u20E3]/gu;
+const WEARABLE_SAFE_AUDIO_FORMAT = Object.freeze({
+  responseFormat: 'wav',
+  mimeType: 'audio/wav',
+  deepgramEncoding: 'linear16',
+  deepgramContainer: 'wav',
+});
 
 function withTimeout(promise, timeoutMs, label) {
   const normalizedTimeout = Number(timeoutMs);
@@ -94,7 +101,7 @@ function resolveApiKey(candidates = []) {
     const snake = lower.replace(/[^a-z0-9]+/g, '_');
     const variants = [key, lower, snake];
     for (const variant of variants) {
-      const value = keys[variant];
+      const value = decryptLocalValue(keys[variant]);
       if (typeof value === 'string' && value.trim()) {
         return value.trim();
       }
@@ -138,9 +145,16 @@ function normalizeVoiceSynthesisOptions(options = {}) {
     provider,
     model: resolveTtsModel(provider, options.model),
     voice: resolveTtsVoice(provider, options.voice),
+    responseFormat: String(options.responseFormat || '').trim().toLowerCase(),
+    transport: String(options.transport || '').trim().toLowerCase(),
   };
 }
 
+function resolveWearableSafeAudioOptions(options = {}) {
+  return String(options.transport || '').trim().toLowerCase() === 'wearable'
+    || String(options.responseFormat || '').trim().toLowerCase() === 'wav';
+}
+
 function requireApiKey(settingLabel, candidates = []) {
   const apiKey = resolveApiKey(candidates);
   if (!apiKey) {
@@ -341,10 +355,15 @@ async function synthesizeWithOpenAi(text, model, voice, options = {}) {
   if (!client) {
     throw new Error('OpenAI TTS is selected but OPENAI_API_KEY is not configured.');
   }
-  const audioBytes = await synthesizeSpeechBuffer(client, text, { model, voice });
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const audioBytes = await synthesizeSpeechBuffer(client, text, {
+    model,
+    voice,
+    responseFormat: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.responseFormat : 'mp3',
+  });
   return {
     audioBytes,
-    mimeType: 'audio/mpeg',
+    mimeType: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
   };
 }
 
@@ -356,25 +375,37 @@ async function streamWithOpenAi(text, model, voice, options = {}, onChunk) {
   if (!client) {
     throw new Error('OpenAI TTS is selected but OPENAI_API_KEY is not configured.');
   }
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
   const response = await client.audio.speech.create({
     model: String(model || 'gpt-4o-mini-tts').trim() || 'gpt-4o-mini-tts',
     voice: String(voice || 'alloy').trim() || 'alloy',
     input: text,
-    response_format: 'mp3',
+    response_format: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.responseFormat : 'mp3',
   });
   const chunks = [];
   for await (const chunk of response.body) {
     chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
   }
   const audioBytes = Buffer.concat(chunks);
-  await onChunk({ audioBytes, mimeType: 'audio/mpeg' });
+  await onChunk({
+    audioBytes,
+    mimeType: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
+  });
 }
 
-async function synthesizeWithDeepgram(text, model) {
+async function synthesizeWithDeepgram(text, model, options = {}) {
   const apiKey = requireApiKey('Deepgram TTS', ['DEEPGRAM_API_KEY']);
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const searchParams = new URLSearchParams({
+    model,
+  });
+  if (useWearableSafeAudio) {
+    searchParams.set('encoding', WEARABLE_SAFE_AUDIO_FORMAT.deepgramEncoding);
+    searchParams.set('container', WEARABLE_SAFE_AUDIO_FORMAT.deepgramContainer);
+  }
 
   return fetchAudioOrThrow(
-    `https://api.deepgram.com/v1/speak?model=${encodeURIComponent(model)}`,
+    `https://api.deepgram.com/v1/speak?${searchParams.toString()}`,
     {
       method: 'POST',
       headers: {
@@ -384,13 +415,22 @@ async function synthesizeWithDeepgram(text, model) {
       body: JSON.stringify({ text }),
     },
     'Deepgram TTS request failed',
+    useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
   );
 }
 
-async function streamWithDeepgram(text, model, onChunk) {
+async function streamWithDeepgram(text, model, options = {}, onChunk) {
   const apiKey = requireApiKey('Deepgram TTS', ['DEEPGRAM_API_KEY']);
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const searchParams = new URLSearchParams({
+    model,
+  });
+  if (useWearableSafeAudio) {
+    searchParams.set('encoding', WEARABLE_SAFE_AUDIO_FORMAT.deepgramEncoding);
+    searchParams.set('container', WEARABLE_SAFE_AUDIO_FORMAT.deepgramContainer);
+  }
   await fetchAudioStreamOrThrow(
-    `https://api.deepgram.com/v1/speak?model=${encodeURIComponent(model)}`,
+    `https://api.deepgram.com/v1/speak?${searchParams.toString()}`,
     {
       method: 'POST',
       headers: {
@@ -400,7 +440,7 @@ async function streamWithDeepgram(text, model, onChunk) {
       body: JSON.stringify({ text }),
     },
     'Deepgram TTS stream failed',
-    'audio/mpeg',
+    useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
     onChunk,
   );
 }
@@ -586,7 +626,7 @@ async function synthesizeVoiceReply(text, options = {}) {
   if (provider === 'openai') {
     request = synthesizeWithOpenAi(content, model, voice, options);
   } else if (provider === 'deepgram') {
-    request = synthesizeWithDeepgram(content, model);
+    request = synthesizeWithDeepgram(content, model, options);
   } else {
     request = synthesizeWithGemini(content, model, voice, options);
   }
@@ -668,7 +708,7 @@ async function synthesizeVoiceReplyStream(text, options = {}, onChunk) {
       if (provider === 'openai') {
         await streamWithOpenAi(chunk, model, voice, options, onChunk);
       } else if (provider === 'deepgram') {
-        await streamWithDeepgram(chunk, model, onChunk);
+        await streamWithDeepgram(chunk, model, options, onChunk);
       } else {
         await streamWithGemini(chunk, model, voice, options, onChunk);
       }

package/server/services/voice/runtimeManager.js

@@ -34,11 +34,17 @@ class VoiceRuntimeManager {
     return this.sessions.get(String(sessionId || '').trim()) || null;
   }
 
-  async openFlutterSession({ userId, agentId = null, socket, sessionId = null } = {}) {
-    if (!socket) {
-      throw new Error('Socket is required to open a Flutter voice session.');
+  async openSession({
+    userId,
+    agentId = null,
+    sessionId = null,
+    platform = 'voice_live',
+    sink,
+    outputMode = 'audio_and_text',
+  } = {}) {
+    if (!sink) {
+      throw new Error('A voice session sink is required.');
     }
-
     const voiceSettings = getVoiceRuntimeSettings(userId, agentId);
     const liveProviderRuntime = this.#getProviderRuntime(
       userId,
@@ -53,16 +59,37 @@ class VoiceRuntimeManager {
       id: resolvedSessionId,
       userId,
       agentId,
-      platform: 'voice_live',
+      platform,
       voiceSettings: {
         ...voiceSettings,
         liveApiKey: liveProviderRuntime.apiKey,
         liveBaseUrl: liveProviderRuntime.baseUrl,
       },
+      sink,
+      outputMode,
+    });
+
+    session.adapter = adapter;
+    this.sessions.set(resolvedSessionId, session);
+    await session.publishReady();
+    return session;
+  }
+
+  async openFlutterSession({ userId, agentId = null, socket, sessionId = null } = {}) {
+    if (!socket) {
+      throw new Error('Socket is required to open a Flutter voice session.');
+    }
+    return this.openSession({
+      userId,
+      agentId,
+      sessionId,
+      platform: 'voice_live',
+      outputMode: 'audio_and_text',
       sink: {
         publishReady: async (_session, extra = {}) => {
+          const voiceSettings = getVoiceRuntimeSettings(userId, agentId);
           socket.emit('voice:session_ready', {
-            sessionId: resolvedSessionId,
+            sessionId: String(sessionId || _session.id).trim(),
             runtimeMode: voiceSettings.runtimeMode,
             provider: voiceSettings.liveProvider,
             model: voiceSettings.liveModel,
@@ -72,52 +99,58 @@ class VoiceRuntimeManager {
         },
         setState: async (_session, state, extra = {}) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state,
             ...extra,
           });
         },
         publishTranscriptPartial: async (_session, content) => {
           socket.emit('voice:transcript_partial', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             content,
           });
         },
         publishTranscriptFinal: async (_session, content) => {
           socket.emit('voice:transcript_final', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             content,
           });
         },
         publishAssistantOutput: async (_session, content, options = {}) => {
-          await this.#deliverFlutterAssistantOutput(socket, resolvedSessionId, session, content, options);
+          await this.#deliverFlutterAssistantOutput(socket, _session.id, _session, content, options);
         },
-        interruptOutput: async () => {
+        interruptOutput: async (_session) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state: 'interrupted',
           });
         },
         publishError: async (_session, message, extra = {}) => {
           socket.emit('voice:error', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             error: message,
             ...extra,
           });
         },
-        close: async () => {
+        close: async (_session) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state: 'closed',
           });
         },
       },
     });
+  }
 
-    session.adapter = adapter;
-    this.sessions.set(resolvedSessionId, session);
-    await session.publishReady();
-    return session;
+  async openWearableSession({ userId, agentId = null, sessionId = null, sink } = {}) {
+    return this.openSession({
+      userId,
+      agentId,
+      sessionId,
+      platform: 'wearable_live',
+      outputMode: 'audio_and_text',
+      sink,
+    });
   }
 
   async closeSession(sessionId, reason = 'closed') {
@@ -435,6 +468,8 @@ class VoiceRuntimeManager {
       provider: session.voiceSettings?.liveProvider,
       model: session.voiceSettings?.liveTtsModel,
       voice: session.voiceSettings?.liveVoice,
+      transport: 'wearable',
+      responseFormat: 'wav',
     });
     const spokenContent = sanitizeSpeechText(content);
 
@@ -490,6 +525,88 @@ class VoiceRuntimeManager {
     }
   }
 
+  async deliverWearableAssistantOutput(ws, sessionId, content, options = {}) {
+    const session = this.getSession(sessionId);
+    if (!session) {
+      return;
+    }
+    const kind = String(options.kind || 'final').trim() || 'final';
+    ws.send(JSON.stringify({
+      type: 'voice:assistant_text',
+      sessionId,
+      content,
+      kind,
+    }));
+
+    if (kind === 'final') {
+      await session.setState('speaking', { kind });
+    }
+
+    const voiceOptions = normalizeVoiceSynthesisOptions({
+      provider: session.voiceSettings?.liveProvider,
+      model: session.voiceSettings?.liveTtsModel,
+      voice: session.voiceSettings?.liveVoice,
+    });
+    const spokenContent = sanitizeSpeechText(content);
+    let index = 0;
+    let streamError = null;
+    const ttsAttempts = this.#buildTtsAttemptOrder(session, voiceOptions);
+
+    if (spokenContent) {
+      try {
+        for (const attempt of ttsAttempts) {
+          index = 0;
+          streamError = null;
+          try {
+            await synthesizeVoiceReplyStream(
+              spokenContent,
+              attempt,
+              async ({ audioBytes, mimeType }) => {
+                if (session.closed || session.interrupted) return;
+                ws.send(JSON.stringify({
+                  type: 'voice:audio_chunk',
+                  sessionId,
+                  kind,
+                  index,
+                  audioBase64: audioBytes.toString('base64'),
+                  mimeType,
+                }));
+                index += 1;
+              },
+            );
+            break;
+          } catch (error) {
+            streamError = String(error?.message || error || 'Voice playback failed.');
+          }
+        }
+      } catch (error) {
+        streamError = String(error?.message || error || 'Voice playback failed.');
+      }
+    }
+
+    if (!streamError && !session.closed && !session.interrupted) {
+      ws.send(JSON.stringify({
+        type: 'voice:audio_done',
+        sessionId,
+        kind,
+        totalChunks: index,
+      }));
+    } else if (kind === 'final' && !session.closed && !session.interrupted) {
+      ws.send(JSON.stringify({
+        type: 'voice:error',
+        sessionId,
+        error: streamError,
+        recoverable: true,
+        phase: 'tts',
+      }));
+      await session.setState('degraded', { kind, phase: 'tts' });
+    }
+
+    if (kind === 'final' && !streamError) {
+      await session.setState('idle');
+    }
+  }
+
   #buildTtsAttemptOrder(session, voiceOptions) {
     const attempts = [];
     const providers = [

package/server/services/voice/turnRunner.js

@@ -110,8 +110,35 @@ async function runVoiceTranscriptTurn({
   });
 
   const replyText = String(runResult?.content || '').trim();
+  const assistantMetadata = {
+    ...persistedMetadata,
+    platform,
+    tokens: runResult?.totalTokens || 0,
+    screenshotIncluded: Boolean(screenshotContext),
+    screenshotVisionProvider: screenshotContext?.provider || null,
+    screenshotVisionModel: screenshotContext?.model || null,
+  };
   if (!replyText) {
-    throw new Error('Agent returned an empty voice reply.');
+    db.prepare('INSERT INTO conversation_history (user_id, agent_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?, ?)')
+      .run(
+        userId,
+        agentId,
+        runResult?.runId || null,
+        'assistant',
+        '',
+        JSON.stringify(assistantMetadata),
+      );
+    return {
+      runId: runResult?.runId || null,
+      transcript: transcriptText,
+      replyText: '',
+      ttsProvider: voiceOptions.provider,
+      ttsModel: voiceOptions.model,
+      ttsVoice: voiceOptions.voice,
+      audioMimeType: 'audio/mpeg',
+      audioBase64: '',
+      ttsError: null,
+    };
   }
 
   db.prepare('INSERT INTO conversation_history (user_id, agent_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?, ?)')
@@ -121,14 +148,7 @@ async function runVoiceTranscriptTurn({
       runResult?.runId || null,
       'assistant',
       replyText,
-      JSON.stringify({
-        ...persistedMetadata,
-        platform,
-        tokens: runResult?.totalTokens || 0,
-        screenshotIncluded: Boolean(screenshotContext),
-        screenshotVisionProvider: screenshotContext?.provider || null,
-        screenshotVisionModel: screenshotContext?.model || null,
-      }),
+      JSON.stringify(assistantMetadata),
     );
 
   let synthesized;