agentgui 1.0.291 → 1.0.293

package/lib/speech.js

@@ -1,18 +1,39 @@
 import { createRequire } from 'module';
 import fs from 'fs';
 import path from 'path';
-import http from 'http';
+import os from 'os';
 import { fileURLToPath } from 'url';
 
 const require = createRequire(import.meta.url);
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT = path.dirname(__dirname);
 
+// Load modules
+let serverTTS = null;
 let serverSTT = null;
+let audioDecode = null;
+let sttttsmodels = null;
+let ttsUtils = null;
+
+try { serverTTS = require('webtalk/server-tts'); } catch(e) { console.warn('[TTS] webtalk/server-tts unavailable:', e.message); }
 try { serverSTT = require('webtalk/server-stt'); } catch(e) { console.warn('[STT] webtalk/server-stt unavailable:', e.message); }
+try { audioDecode = require('audio-decode'); } catch(e) { console.warn('[TTS] audio-decode unavailable:', e.message); }
+try { sttttsmodels = require('sttttsmodels'); } catch(e) { console.warn('[TTS] sttttsmodels unavailable:', e.message); }
+try { ttsUtils = require('webtalk/tts-utils'); } catch(e) {}
+
+// Detect webtalk API type: old (server-tts.js with getVoices/synthesizeViaPocket)
+// vs new ONNX (server-tts-onnx.js with encodeVoiceAudio)
+const isOnnxApi = serverTTS && typeof serverTTS.encodeVoiceAudio === 'function';
+const isPocketApi = serverTTS && typeof serverTTS.getVoices === 'function';
+
+// Voice directories to scan
+const VOICE_DIRS = [
+  path.join(os.homedir(), 'voices'),
+  path.join(ROOT, 'voices'),
+  '/config/voices',
+];
 
-const VOICE_DIRS = [path.join(ROOT, 'voices')];
-const POCKET_PORT = 8787;
+const AUDIO_EXTENSIONS = ['.wav', '.mp3', '.ogg', '.flac', '.m4a'];
 
 const POCKET_TTS_VOICES = [
   { id: 'default', name: 'Default', gender: 'female', accent: 'French' },
@@ -26,7 +47,104 @@ const POCKET_TTS_VOICES = [
   { id: 'azelma', name: 'Azelma', gender: 'female', accent: 'French' },
 ];
 
-const PREDEFINED_IDS = new Set(POCKET_TTS_VOICES.filter(v => v.id !== 'default').map(v => v.id));
+const SAMPLE_RATE = 24000;
+
+// Embedding cache: voiceId -> {data, shape}
+const voiceEmbeddingCache = new Map();
+
+function getModelDir() {
+  if (sttttsmodels && sttttsmodels.ttsDir && fs.existsSync(sttttsmodels.ttsDir)) {
+    return sttttsmodels.ttsDir;
+  }
+  // Fallback to persistent cache dir
+  return path.join(os.homedir(), '.gmgui', 'models', 'tts');
+}
+
+function findVoiceFile(voiceId) {
+  if (!voiceId || voiceId === 'default') return null;
+  const baseName = voiceId.replace(/^custom_/, '');
+  for (const dir of VOICE_DIRS) {
+    for (const ext of AUDIO_EXTENSIONS) {
+      const p = path.join(dir, baseName + ext);
+      if (fs.existsSync(p)) return p;
+    }
+  }
+  return null;
+}
+
+function scanVoiceDir(dir) {
+  const voices = [];
+  try {
+    if (!fs.existsSync(dir)) return voices;
+    const seen = new Set();
+    for (const file of fs.readdirSync(dir)) {
+      const ext = path.extname(file).toLowerCase();
+      if (!AUDIO_EXTENSIONS.includes(ext)) continue;
+      const baseName = path.basename(file, ext);
+      if (seen.has(baseName)) continue;
+      seen.add(baseName);
+      voices.push({
+        id: 'custom_' + baseName.replace(/[^a-zA-Z0-9_-]/g, '_'),
+        name: baseName.replace(/_/g, ' '),
+        gender: 'custom', accent: 'custom', isCustom: true,
+      });
+    }
+  } catch (_) {}
+  return voices;
+}
+
+// Encode a voice WAV file to an ONNX voice embedding
+async function getVoiceEmbedding(voiceId) {
+  if (voiceEmbeddingCache.has(voiceId)) return voiceEmbeddingCache.get(voiceId);
+  const voicePath = findVoiceFile(voiceId);
+  if (!voicePath) return null;
+  if (!audioDecode || !serverTTS || !isOnnxApi) return null;
+
+  const modelDir = getModelDir();
+  if (serverTTS.loadModels) await serverTTS.loadModels(modelDir);
+
+  const raw = fs.readFileSync(voicePath);
+  const decoded = await audioDecode.default(raw);
+  let pcm = decoded.getChannelData(0);
+  if (decoded.sampleRate !== SAMPLE_RATE) {
+    pcm = ttsUtils ? ttsUtils.resample(pcm, decoded.sampleRate, SAMPLE_RATE)
+      : (() => {
+        const ratio = decoded.sampleRate / SAMPLE_RATE;
+        const out = new Float32Array(Math.round(pcm.length / ratio));
+        for (let i = 0; i < out.length; i++) out[i] = pcm[Math.floor(i * ratio)];
+        return out;
+      })();
+  }
+
+  const embedding = await serverTTS.encodeVoiceAudio(pcm);
+  voiceEmbeddingCache.set(voiceId, embedding);
+  return embedding;
+}
+
+// Convert Float32Array PCM to WAV buffer
+function pcmToWav(samples, sampleRate = SAMPLE_RATE) {
+  const numSamples = samples.length;
+  const numChannels = 1;
+  const bitsPerSample = 16;
+  const byteRate = sampleRate * numChannels * bitsPerSample / 8;
+  const blockAlign = numChannels * bitsPerSample / 8;
+  const dataSize = numSamples * blockAlign;
+  const buf = Buffer.alloc(44 + dataSize);
+
+  buf.write('RIFF', 0); buf.writeUInt32LE(36 + dataSize, 4);
+  buf.write('WAVE', 8); buf.write('fmt ', 12);
+  buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20);
+  buf.writeUInt16LE(numChannels, 22); buf.writeUInt32LE(sampleRate, 24);
+  buf.writeUInt32LE(byteRate, 28); buf.writeUInt16LE(blockAlign, 32);
+  buf.writeUInt16LE(bitsPerSample, 34); buf.write('data', 36);
+  buf.writeUInt32LE(dataSize, 40);
+
+  for (let i = 0; i < numSamples; i++) {
+    const s = Math.max(-1, Math.min(1, samples[i]));
+    buf.writeInt16LE(Math.round(s * 32767), 44 + i * 2);
+  }
+  return buf;
+}
 
 function getSttOptions() {
   if (process.env.PORTABLE_EXE_DIR) {
@@ -38,56 +156,40 @@ function getSttOptions() {
   return {};
 }
 
-function findVoiceFile(voiceId) {
-  for (const dir of VOICE_DIRS) {
-    const p = path.join(dir, `custom_${voiceId}.wav`);
-    if (fs.existsSync(p)) return p;
+async function synthesize(text, voiceId) {
+  if (isOnnxApi) {
+    // Node.js ONNX TTS - no Python required
+    const modelDir = getModelDir();
+    const embedding = voiceId ? await getVoiceEmbedding(voiceId) : null;
+    const pcm = await serverTTS.synthesize(text, embedding, modelDir);
+    return pcmToWav(pcm);
   }
-  return null;
-}
 
-function synthesize(text, voiceId) {
-  const voicePath = voiceId ? findVoiceFile(voiceId) : null;
-  const isPredefined = voiceId && PREDEFINED_IDS.has(voiceId);
-  const boundary = '----PocketTTS' + Date.now();
-  const parts = [];
-  parts.push(`--${boundary}\r\nContent-Disposition: form-data; name="text"\r\n\r\n${text}\r\n`);
-  if (voicePath) {
-    const data = fs.readFileSync(voicePath);
-    const name = path.basename(voicePath);
-    parts.push(`--${boundary}\r\nContent-Disposition: form-data; name="voice_wav"; filename="${name}"\r\nContent-Type: audio/wav\r\n\r\n`);
-    parts.push(data);
-    parts.push('\r\n');
-  } else if (isPredefined) {
-    parts.push(`--${boundary}\r\nContent-Disposition: form-data; name="voice_url"\r\n\r\n${voiceId}\r\n`);
+  if (isPocketApi) {
+    // Old server-tts.js with pocket-tts sidecar
+    return serverTTS.synthesize(text, voiceId, VOICE_DIRS);
   }
-  parts.push(`--${boundary}--\r\n`);
-  const body = Buffer.concat(parts.map(p => Buffer.isBuffer(p) ? p : Buffer.from(p)));
-  return new Promise((resolve, reject) => {
-    const req = http.request({
-      hostname: '127.0.0.1', port: POCKET_PORT, path: '/tts', method: 'POST',
-      headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}`, 'Content-Length': body.length },
-      timeout: 60000,
-    }, res => {
-      if (res.statusCode !== 200) {
-        let e = '';
-        res.on('data', d => e += d);
-        res.on('end', () => reject(new Error(`pocket-tts HTTP ${res.statusCode}: ${e}`)));
-        return;
-      }
-      const chunks = [];
-      res.on('data', d => chunks.push(d));
-      res.on('end', () => resolve(Buffer.concat(chunks)));
-    });
-    req.on('error', reject);
-    req.on('timeout', () => { req.destroy(); reject(new Error('pocket-tts timeout')); });
-    req.write(body);
-    req.end();
-  });
+
+  throw new Error('No TTS backend available');
 }
 
 async function* synthesizeStream(text, voiceId) {
-  yield await synthesize(text, voiceId);
+  if (isOnnxApi) {
+    const modelDir = getModelDir();
+    const embedding = voiceId ? await getVoiceEmbedding(voiceId) : null;
+    const pcm = await serverTTS.synthesize(text, embedding, modelDir);
+    yield pcmToWav(pcm);
+    return;
+  }
+
+  if (isPocketApi) {
+    for await (const chunk of serverTTS.synthesizeStream(text, voiceId, VOICE_DIRS)) {
+      yield chunk;
+    }
+    return;
+  }
+
+  throw new Error('No TTS backend available');
 }
 
 function transcribe(audioBuffer) {
@@ -101,29 +203,61 @@ function getSTT() {
 }
 
 function getVoices() {
-  return POCKET_TTS_VOICES;
+  const seen = new Set();
+  const custom = [];
+  for (const dir of VOICE_DIRS) {
+    for (const v of scanVoiceDir(dir)) {
+      if (seen.has(v.id)) continue;
+      seen.add(v.id);
+      custom.push(v);
+    }
+  }
+  // Include built-in voices from old server-tts if available
+  if (isPocketApi) {
+    const upstream = serverTTS.getVoices(VOICE_DIRS).filter(v => v.isCustom);
+    for (const v of upstream) {
+      if (!seen.has(v.id)) { seen.add(v.id); custom.push(v); }
+    }
+  }
+  return [...POCKET_TTS_VOICES, ...custom];
 }
 
 function getStatus() {
   const sttStatus = serverSTT ? serverSTT.getStatus() : { ready: false, loading: false, error: 'STT unavailable' };
+  const ttsBackend = isOnnxApi ? 'onnx-node' : isPocketApi ? 'pocket-tts' : 'none';
   return {
     sttReady: sttStatus.ready,
-    ttsReady: true,
+    ttsReady: isOnnxApi || isPocketApi,
     sttLoading: sttStatus.loading,
     ttsLoading: false,
     sttError: sttStatus.error,
-    ttsError: null,
+    ttsError: (!isOnnxApi && !isPocketApi) ? 'No TTS backend available' : null,
+    ttsBackend,
   };
 }
 
 function preloadTTS() {
-  // pocket-tts is managed externally; nothing to preload
+  if (isOnnxApi) {
+    // Pre-load ONNX models in background
+    const modelDir = getModelDir();
+    if (serverTTS.loadModels) {
+      serverTTS.loadModels(modelDir).catch(e => console.warn('[TTS] ONNX preload failed:', e.message));
+    }
+  } else if (isPocketApi && serverTTS.preload) {
+    serverTTS.preload(null, {});
+  }
 }
 
-function ttsCacheKey(text, voiceId) { return null; }
-function ttsCacheGet(key) { return null; }
+function ttsCacheKey(text, voiceId) {
+  return isPocketApi && serverTTS.ttsCacheKey ? serverTTS.ttsCacheKey(text, voiceId) : null;
+}
+
+function ttsCacheGet(key) {
+  return isPocketApi && serverTTS.ttsCacheGet ? serverTTS.ttsCacheGet(key) : null;
+}
 
 function splitSentences(text) {
+  if (isPocketApi && serverTTS.splitSentences) return serverTTS.splitSentences(text);
   return text.match(/[^.!?]+[.!?]*/g)?.map(s => s.trim()).filter(Boolean) || [text];
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgui",
-  "version": "1.0.291",
+  "version": "1.0.293",
   "description": "Multi-agent ACP client with real-time communication",
   "type": "module",
   "main": "server.js",