neoagent 2.3.1-beta.6 → 2.3.1-beta.60

package/server/services/voice/providers.js

@@ -6,6 +6,7 @@ const { AGENT_DATA_DIR } = require('../../../runtime/paths');
 const { getOpenAiClient } = require('./openaiClient');
 const { synthesizeSpeechBuffer } = require('./openaiSpeech');
 const { transcribeChunkWithDeepgram } = require('../recordings/deepgram');
+const { decryptLocalValue } = require('../../utils/local_secrets');
 
 const DEFAULT_STT_PROVIDER = 'openai';
 const DEFAULT_TTS_PROVIDER = 'openai';
@@ -36,6 +37,20 @@ const DEFAULT_GEMINI_TRANSCRIPTION_PROMPT =
   'Transcribe this audio verbatim. Return only the transcript text.';
 const EMOJI_SPEECH_REGEX =
   /[\p{Extended_Pictographic}\p{Emoji_Presentation}\p{Regional_Indicator}\u200D\uFE0F\u20E3]/gu;
+const WEARABLE_SAFE_AUDIO_FORMAT = Object.freeze({
+  responseFormat: 'wav',
+  mimeType: 'audio/wav',
+  streamResponseFormat: 'pcm',
+  streamMimeType: 'audio/wav',
+  pcmSampleRate: 24000,
+  pcmChannels: 1,
+  pcmBitsPerSample: 16,
+  deepgramEncoding: 'linear16',
+  deepgramContainer: 'wav',
+  deepgramStreamContainer: 'none',
+});
+const MIN_STREAM_PCM_CHUNK_BYTES = 24000;
+const MAX_STREAM_PCM_CHUNK_BYTES = 48000;
 
 function withTimeout(promise, timeoutMs, label) {
   const normalizedTimeout = Number(timeoutMs);
@@ -94,7 +109,7 @@ function resolveApiKey(candidates = []) {
     const snake = lower.replace(/[^a-z0-9]+/g, '_');
     const variants = [key, lower, snake];
     for (const variant of variants) {
-      const value = keys[variant];
+      const value = decryptLocalValue(keys[variant]);
       if (typeof value === 'string' && value.trim()) {
         return value.trim();
       }
@@ -138,9 +153,16 @@ function normalizeVoiceSynthesisOptions(options = {}) {
     provider,
     model: resolveTtsModel(provider, options.model),
     voice: resolveTtsVoice(provider, options.voice),
+    responseFormat: String(options.responseFormat || '').trim().toLowerCase(),
+    transport: String(options.transport || '').trim().toLowerCase(),
   };
 }
 
+function resolveWearableSafeAudioOptions(options = {}) {
+  return String(options.transport || '').trim().toLowerCase() === 'wearable'
+    || String(options.responseFormat || '').trim().toLowerCase() === 'wav';
+}
+
 function requireApiKey(settingLabel, candidates = []) {
   const apiKey = resolveApiKey(candidates);
   if (!apiKey) {
@@ -184,10 +206,11 @@ async function fetchAudioStreamOrThrow(url, init, errorPrefix, defaultMimeType =
   while (true) {
     const { done, value } = await reader.read();
     if (done) break;
-    if (value?.length) chunks.push(Buffer.from(value));
+    if (value?.length) {
+      chunks.push(Buffer.from(value));
+    }
   }
-  const audioBytes = Buffer.concat(chunks);
-  await onChunk({ audioBytes, mimeType });
+  await onChunk({ audioBytes: Buffer.concat(chunks), mimeType });
 }
 
 function guessExtFromMimeType(mimeType) {
@@ -248,6 +271,49 @@ function wrapPcmAsWav(audioBytes, format) {
   return Buffer.concat([header, data]);
 }
 
+async function streamPcmAsWavChunks(readable, format, onChunk) {
+  const source = readable && typeof readable.getReader === 'function'
+    ? readable
+    : null;
+  let pending = Buffer.alloc(0);
+
+  async function flushPending(force = false) {
+    while (pending.length >= MIN_STREAM_PCM_CHUNK_BYTES || (force && pending.length > 0)) {
+      const targetLength = force
+        ? pending.length
+        : Math.min(pending.length, MAX_STREAM_PCM_CHUNK_BYTES);
+      const evenLength = targetLength - (targetLength % 2);
+      if (evenLength <= 0) return;
+      const pcmChunk = pending.subarray(0, evenLength);
+      pending = pending.subarray(evenLength);
+      await onChunk({
+        audioBytes: wrapPcmAsWav(pcmChunk, format),
+        mimeType: WEARABLE_SAFE_AUDIO_FORMAT.streamMimeType,
+      });
+    }
+  }
+
+  if (source) {
+    const reader = source.getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value?.length) {
+        pending = Buffer.concat([pending, Buffer.from(value)]);
+        await flushPending(false);
+      }
+    }
+  } else {
+    for await (const chunk of readable) {
+      if (chunk?.length) {
+        pending = Buffer.concat([pending, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]);
+        await flushPending(false);
+      }
+    }
+  }
+  await flushPending(true);
+}
+
 async function transcribeWithOpenAi(filePath, model, options = {}) {
   const client = getOpenAiClient({
     apiKey: typeof options.apiKey === 'string' ? options.apiKey.trim() : '',
@@ -341,10 +407,15 @@ async function synthesizeWithOpenAi(text, model, voice, options = {}) {
   if (!client) {
     throw new Error('OpenAI TTS is selected but OPENAI_API_KEY is not configured.');
   }
-  const audioBytes = await synthesizeSpeechBuffer(client, text, { model, voice });
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const audioBytes = await synthesizeSpeechBuffer(client, text, {
+    model,
+    voice,
+    responseFormat: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.responseFormat : 'mp3',
+  });
   return {
     audioBytes,
-    mimeType: 'audio/mpeg',
+    mimeType: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
   };
 }
 
@@ -356,25 +427,49 @@ async function streamWithOpenAi(text, model, voice, options = {}, onChunk) {
   if (!client) {
     throw new Error('OpenAI TTS is selected but OPENAI_API_KEY is not configured.');
   }
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
   const response = await client.audio.speech.create({
     model: String(model || 'gpt-4o-mini-tts').trim() || 'gpt-4o-mini-tts',
     voice: String(voice || 'alloy').trim() || 'alloy',
     input: text,
-    response_format: 'mp3',
+    response_format: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.streamResponseFormat : 'mp3',
   });
+  if (useWearableSafeAudio) {
+    await streamPcmAsWavChunks(
+      response.body,
+      {
+        bitsPerSample: WEARABLE_SAFE_AUDIO_FORMAT.pcmBitsPerSample,
+        sampleRate: WEARABLE_SAFE_AUDIO_FORMAT.pcmSampleRate,
+        channels: WEARABLE_SAFE_AUDIO_FORMAT.pcmChannels,
+      },
+      onChunk,
+    );
+    return;
+  }
   const chunks = [];
   for await (const chunk of response.body) {
     chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
   }
   const audioBytes = Buffer.concat(chunks);
-  await onChunk({ audioBytes, mimeType: 'audio/mpeg' });
+  await onChunk({
+    audioBytes,
+    mimeType: useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
+  });
 }
 
-async function synthesizeWithDeepgram(text, model) {
+async function synthesizeWithDeepgram(text, model, options = {}) {
   const apiKey = requireApiKey('Deepgram TTS', ['DEEPGRAM_API_KEY']);
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const searchParams = new URLSearchParams({
+    model,
+  });
+  if (useWearableSafeAudio) {
+    searchParams.set('encoding', WEARABLE_SAFE_AUDIO_FORMAT.deepgramEncoding);
+    searchParams.set('container', WEARABLE_SAFE_AUDIO_FORMAT.deepgramContainer);
+  }
 
   return fetchAudioOrThrow(
-    `https://api.deepgram.com/v1/speak?model=${encodeURIComponent(model)}`,
+    `https://api.deepgram.com/v1/speak?${searchParams.toString()}`,
     {
       method: 'POST',
       headers: {
@@ -384,13 +479,47 @@ async function synthesizeWithDeepgram(text, model) {
       body: JSON.stringify({ text }),
     },
     'Deepgram TTS request failed',
+    useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
   );
 }
 
-async function streamWithDeepgram(text, model, onChunk) {
+async function streamWithDeepgram(text, model, options = {}, onChunk) {
   const apiKey = requireApiKey('Deepgram TTS', ['DEEPGRAM_API_KEY']);
+  const useWearableSafeAudio = resolveWearableSafeAudioOptions(options);
+  const searchParams = new URLSearchParams({
+    model,
+  });
+  if (useWearableSafeAudio) {
+    searchParams.set('encoding', WEARABLE_SAFE_AUDIO_FORMAT.deepgramEncoding);
+    searchParams.set('container', WEARABLE_SAFE_AUDIO_FORMAT.deepgramStreamContainer);
+    searchParams.set('sample_rate', String(WEARABLE_SAFE_AUDIO_FORMAT.pcmSampleRate));
+    const response = await fetch(
+      `https://api.deepgram.com/v1/speak?${searchParams.toString()}`,
+      {
+        method: 'POST',
+        headers: {
+          Authorization: `Token ${apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ text }),
+      },
+    );
+    if (!response.ok) {
+      await throwResponseError(response, 'Deepgram TTS stream failed');
+    }
+    await streamPcmAsWavChunks(
+      response.body,
+      {
+        bitsPerSample: WEARABLE_SAFE_AUDIO_FORMAT.pcmBitsPerSample,
+        sampleRate: WEARABLE_SAFE_AUDIO_FORMAT.pcmSampleRate,
+        channels: WEARABLE_SAFE_AUDIO_FORMAT.pcmChannels,
+      },
+      onChunk,
+    );
+    return;
+  }
   await fetchAudioStreamOrThrow(
-    `https://api.deepgram.com/v1/speak?model=${encodeURIComponent(model)}`,
+    `https://api.deepgram.com/v1/speak?${searchParams.toString()}`,
     {
       method: 'POST',
       headers: {
@@ -400,7 +529,7 @@ async function streamWithDeepgram(text, model, onChunk) {
       body: JSON.stringify({ text }),
     },
     'Deepgram TTS stream failed',
-    'audio/mpeg',
+    useWearableSafeAudio ? WEARABLE_SAFE_AUDIO_FORMAT.mimeType : 'audio/mpeg',
     onChunk,
   );
 }
@@ -586,7 +715,7 @@ async function synthesizeVoiceReply(text, options = {}) {
   if (provider === 'openai') {
     request = synthesizeWithOpenAi(content, model, voice, options);
   } else if (provider === 'deepgram') {
-    request = synthesizeWithDeepgram(content, model);
+    request = synthesizeWithDeepgram(content, model, options);
   } else {
     request = synthesizeWithGemini(content, model, voice, options);
   }
@@ -668,7 +797,7 @@ async function synthesizeVoiceReplyStream(text, options = {}, onChunk) {
       if (provider === 'openai') {
         await streamWithOpenAi(chunk, model, voice, options, onChunk);
       } else if (provider === 'deepgram') {
-        await streamWithDeepgram(chunk, model, onChunk);
+        await streamWithDeepgram(chunk, model, options, onChunk);
       } else {
         await streamWithGemini(chunk, model, voice, options, onChunk);
       }

package/server/services/voice/runtimeManager.js

@@ -34,11 +34,17 @@ class VoiceRuntimeManager {
     return this.sessions.get(String(sessionId || '').trim()) || null;
   }
 
-  async openFlutterSession({ userId, agentId = null, socket, sessionId = null } = {}) {
-    if (!socket) {
-      throw new Error('Socket is required to open a Flutter voice session.');
+  async openSession({
+    userId,
+    agentId = null,
+    sessionId = null,
+    platform = 'voice_live',
+    sink,
+    outputMode = 'audio_and_text',
+  } = {}) {
+    if (!sink) {
+      throw new Error('A voice session sink is required.');
     }
-
     const voiceSettings = getVoiceRuntimeSettings(userId, agentId);
     const liveProviderRuntime = this.#getProviderRuntime(
       userId,
@@ -53,16 +59,37 @@ class VoiceRuntimeManager {
       id: resolvedSessionId,
       userId,
       agentId,
-      platform: 'voice_live',
+      platform,
       voiceSettings: {
         ...voiceSettings,
         liveApiKey: liveProviderRuntime.apiKey,
         liveBaseUrl: liveProviderRuntime.baseUrl,
       },
+      sink,
+      outputMode,
+    });
+
+    session.adapter = adapter;
+    this.sessions.set(resolvedSessionId, session);
+    await session.publishReady();
+    return session;
+  }
+
+  async openFlutterSession({ userId, agentId = null, socket, sessionId = null } = {}) {
+    if (!socket) {
+      throw new Error('Socket is required to open a Flutter voice session.');
+    }
+    return this.openSession({
+      userId,
+      agentId,
+      sessionId,
+      platform: 'voice_live',
+      outputMode: 'audio_and_text',
       sink: {
         publishReady: async (_session, extra = {}) => {
+          const voiceSettings = getVoiceRuntimeSettings(userId, agentId);
           socket.emit('voice:session_ready', {
-            sessionId: resolvedSessionId,
+            sessionId: String(sessionId || _session.id).trim(),
             runtimeMode: voiceSettings.runtimeMode,
             provider: voiceSettings.liveProvider,
             model: voiceSettings.liveModel,
@@ -72,52 +99,58 @@ class VoiceRuntimeManager {
         },
         setState: async (_session, state, extra = {}) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state,
             ...extra,
           });
         },
         publishTranscriptPartial: async (_session, content) => {
           socket.emit('voice:transcript_partial', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             content,
           });
         },
         publishTranscriptFinal: async (_session, content) => {
           socket.emit('voice:transcript_final', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             content,
           });
         },
         publishAssistantOutput: async (_session, content, options = {}) => {
-          await this.#deliverFlutterAssistantOutput(socket, resolvedSessionId, session, content, options);
+          await this.#deliverFlutterAssistantOutput(socket, _session.id, _session, content, options);
         },
-        interruptOutput: async () => {
+        interruptOutput: async (_session) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state: 'interrupted',
           });
         },
         publishError: async (_session, message, extra = {}) => {
           socket.emit('voice:error', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             error: message,
             ...extra,
           });
         },
-        close: async () => {
+        close: async (_session) => {
           socket.emit('voice:assistant_state', {
-            sessionId: resolvedSessionId,
+            sessionId: _session.id,
             state: 'closed',
           });
         },
       },
     });
+  }
 
-    session.adapter = adapter;
-    this.sessions.set(resolvedSessionId, session);
-    await session.publishReady();
-    return session;
+  async openWearableSession({ userId, agentId = null, sessionId = null, sink } = {}) {
+    return this.openSession({
+      userId,
+      agentId,
+      sessionId,
+      platform: 'wearable_live',
+      outputMode: 'audio_and_text',
+      sink,
+    });
   }
 
   async closeSession(sessionId, reason = 'closed') {
@@ -446,6 +479,7 @@ class VoiceRuntimeManager {
       for (const attempt of ttsAttempts) {
         index = 0;
         streamError = null;
+        let attemptChunks = 0;
         try {
           await synthesizeVoiceReplyStream(
             spokenContent,
@@ -459,13 +493,21 @@ class VoiceRuntimeManager {
                 audioBase64: audioBytes.toString('base64'),
                 mimeType,
               });
+              attemptChunks += 1;
               index += 1;
             },
           );
+          if (attemptChunks === 0) {
+            throw new Error(`${attempt.provider} TTS produced no audio chunks.`);
+          }
           streamError = null;
           break;
         } catch (error) {
           streamError = String(error?.message || error || 'Voice playback failed.');
+          console.warn(`[VoiceRuntime] ${attempt.provider} TTS failed for flutter session ${sessionId}: ${streamError}`);
+          if (attemptChunks > 0) {
+            break;
+          }
         }
       }
       } catch (error) {
@@ -490,6 +532,99 @@ class VoiceRuntimeManager {
     }
   }
 
+  async deliverWearableAssistantOutput(ws, sessionId, content, options = {}) {
+    const session = this.getSession(sessionId);
+    if (!session) {
+      return;
+    }
+    const kind = String(options.kind || 'final').trim() || 'final';
+    ws.send(JSON.stringify({
+      type: 'voice:assistant_text',
+      sessionId,
+      content,
+      kind,
+    }));
+
+    if (kind === 'final') {
+      await session.setState('speaking', { kind });
+    }
+
+    const voiceOptions = normalizeVoiceSynthesisOptions({
+      provider: session.voiceSettings?.liveProvider,
+      model: session.voiceSettings?.liveTtsModel,
+      voice: session.voiceSettings?.liveVoice,
+      transport: 'wearable',
+      responseFormat: 'wav',
+    });
+    const spokenContent = sanitizeSpeechText(content);
+    let index = 0;
+    let streamError = null;
+    const ttsAttempts = this.#buildTtsAttemptOrder(session, voiceOptions);
+
+    if (spokenContent) {
+      try {
+        for (const attempt of ttsAttempts) {
+          index = 0;
+          streamError = null;
+          let attemptChunks = 0;
+          try {
+            await synthesizeVoiceReplyStream(
+              spokenContent,
+              attempt,
+              async ({ audioBytes, mimeType }) => {
+                if (session.closed || session.interrupted) return;
+                ws.send(JSON.stringify({
+                  type: 'voice:audio_chunk',
+                  sessionId,
+                  kind,
+                  index,
+                  audioBase64: audioBytes.toString('base64'),
+                  mimeType,
+                }));
+                attemptChunks += 1;
+                index += 1;
+              },
+            );
+            if (attemptChunks === 0) {
+              throw new Error(`${attempt.provider} TTS produced no audio chunks.`);
+            }
+            break;
+          } catch (error) {
+            streamError = String(error?.message || error || 'Voice playback failed.');
+            console.warn(`[VoiceRuntime] ${attempt.provider} TTS failed for wearable session ${sessionId}: ${streamError}`);
+            if (attemptChunks > 0) {
+              break;
+            }
+          }
+        }
+      } catch (error) {
+        streamError = String(error?.message || error || 'Voice playback failed.');
+      }
+    }
+
+    if (!streamError && !session.closed && !session.interrupted) {
+      ws.send(JSON.stringify({
+        type: 'voice:audio_done',
+        sessionId,
+        kind,
+        totalChunks: index,
+      }));
+    } else if (kind === 'final' && !session.closed && !session.interrupted) {
+      ws.send(JSON.stringify({
+        type: 'voice:error',
+        sessionId,
+        error: streamError,
+        recoverable: true,
+        phase: 'tts',
+      }));
+      await session.setState('degraded', { kind, phase: 'tts' });
+    }
+
+    if (kind === 'final' && !streamError) {
+      await session.setState('idle');
+    }
+  }
+
   #buildTtsAttemptOrder(session, voiceOptions) {
     const attempts = [];
     const providers = [
@@ -501,6 +636,8 @@ class VoiceRuntimeManager {
         provider,
         model: provider === voiceOptions.provider ? voiceOptions.model : null,
         voice: provider === voiceOptions.provider ? voiceOptions.voice : null,
+        transport: voiceOptions.transport,
+        responseFormat: voiceOptions.responseFormat,
       });
       const runtime = provider === voiceOptions.provider
         ? {

package/server/services/voice/turnRunner.js

@@ -110,8 +110,35 @@ async function runVoiceTranscriptTurn({
   });
 
   const replyText = String(runResult?.content || '').trim();
+  const assistantMetadata = {
+    ...persistedMetadata,
+    platform,
+    tokens: runResult?.totalTokens || 0,
+    screenshotIncluded: Boolean(screenshotContext),
+    screenshotVisionProvider: screenshotContext?.provider || null,
+    screenshotVisionModel: screenshotContext?.model || null,
+  };
   if (!replyText) {
-    throw new Error('Agent returned an empty voice reply.');
+    db.prepare('INSERT INTO conversation_history (user_id, agent_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?, ?)')
+      .run(
+        userId,
+        agentId,
+        runResult?.runId || null,
+        'assistant',
+        '',
+        JSON.stringify(assistantMetadata),
+      );
+    return {
+      runId: runResult?.runId || null,
+      transcript: transcriptText,
+      replyText: '',
+      ttsProvider: voiceOptions.provider,
+      ttsModel: voiceOptions.model,
+      ttsVoice: voiceOptions.voice,
+      audioMimeType: 'audio/mpeg',
+      audioBase64: '',
+      ttsError: null,
+    };
   }
 
   db.prepare('INSERT INTO conversation_history (user_id, agent_id, agent_run_id, role, content, metadata) VALUES (?, ?, ?, ?, ?, ?)')
@@ -121,14 +148,7 @@ async function runVoiceTranscriptTurn({
       runResult?.runId || null,
       'assistant',
       replyText,
-      JSON.stringify({
-        ...persistedMetadata,
-        platform,
-        tokens: runResult?.totalTokens || 0,
-        screenshotIncluded: Boolean(screenshotContext),
-        screenshotVisionProvider: screenshotContext?.provider || null,
-        screenshotVisionModel: screenshotContext?.model || null,
-      }),
+      JSON.stringify(assistantMetadata),
     );
 
   let synthesized;