agentgui 1.0.168 → 1.0.170

package/lib/speech.js

@@ -1,5 +1,6 @@
 import { createRequire } from 'module';
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
@@ -7,19 +8,74 @@ const require = createRequire(import.meta.url);
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT = path.dirname(__dirname);
 const DATA_DIR = path.join(ROOT, 'data');
+const VOICES_DIR = path.join(ROOT, 'voices');
+const HOME_VOICES_DIR = path.join(os.homedir(), 'voices');
+const AUDIO_EXTENSIONS = ['.mp3', '.wav', '.ogg', '.flac', '.m4a'];
+const MIN_WAV_SIZE = 1000;
 
-const SPEAKER_EMBEDDINGS_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
+const BASE_VOICES = [
+  { id: 'default', name: 'Default', gender: 'male', accent: 'US' },
+  { id: 'bdl', name: 'BDL', gender: 'male', accent: 'US' },
+  { id: 'slt', name: 'SLT', gender: 'female', accent: 'US' },
+  { id: 'clb', name: 'CLB', gender: 'female', accent: 'US' },
+  { id: 'rms', name: 'RMS', gender: 'male', accent: 'US' },
+  { id: 'awb', name: 'AWB', gender: 'male', accent: 'Scottish' },
+  { id: 'jmk', name: 'JMK', gender: 'male', accent: 'Canadian' },
+  { id: 'ksp', name: 'KSP', gender: 'male', accent: 'Indian' },
+];
+
+function scanVoiceDir(dir) {
+  const voices = [];
+  try {
+    if (!fs.existsSync(dir)) return voices;
+    for (const file of fs.readdirSync(dir)) {
+      const ext = path.extname(file).toLowerCase();
+      if (!AUDIO_EXTENSIONS.includes(ext)) continue;
+      const baseName = path.basename(file, ext);
+      const id = 'custom_' + baseName.replace(/[^a-zA-Z0-9_-]/g, '_');
+      const name = baseName.replace(/_/g, ' ');
+      voices.push({ id, name, gender: 'custom', accent: 'custom', isCustom: true, sourceDir: dir });
+    }
+  } catch (err) {
+    console.error('[VOICES] Error scanning', dir + ':', err.message);
+  }
+  return voices;
+}
+
+function loadCustomVoices() {
+  const seen = new Set();
+  const voices = [];
+  for (const dir of [VOICES_DIR, HOME_VOICES_DIR]) {
+    for (const v of scanVoiceDir(dir)) {
+      if (seen.has(v.id)) continue;
+      seen.add(v.id);
+      voices.push(v);
+    }
+  }
+  return voices;
+}
+
+function getVoices() {
+  return [...BASE_VOICES, ...loadCustomVoices()];
+}
+
+const SPEAKER_OFFSETS = { awb: 0, bdl: 1200, clb: 2300, jmk: 3500, ksp: 4700, rms: 5900, slt: 7100 };
+const SPEAKER_EMBEDDINGS_URL = 'https://huggingface.co/datasets/Xenova/speaker_embeddings/resolve/main/spkrec-xvectors-voxceleb.hf';
 const SPEAKER_EMBEDDINGS_PATH = path.join(DATA_DIR, 'speaker_embeddings.bin');
-const SAMPLE_RATE_TTS = 16000;
-const SAMPLE_RATE_STT = 16000;
-const MIN_WAV_SIZE = 44;
+const DATASET_API = 'https://datasets-server.huggingface.co/rows?dataset=Xenova%2Fspeaker_embeddings&config=default&split=train';
+const SAMPLES_TO_AVERAGE = 30;
 
 let transformersModule = null;
 let sttPipeline = null;
 let ttsPipeline = null;
 let speakerEmbeddings = null;
+let speakerEmbeddingPipeline = null;
 let sttLoading = false;
 let ttsLoading = false;
+let speakerEmbeddingLoading = false;
+const voiceEmbeddingsCache = new Map();
+const SAMPLE_RATE_STT = 16000;
+const SAMPLE_RATE_TTS = 16000;
 
 const TTS_CACHE_MAX = 100;
 const ttsCache = new Map();
@@ -52,6 +108,115 @@ async function ensureSpeakerEmbeddings() {
   return speakerEmbeddings;
 }
 
+async function loadVoiceEmbedding(voiceId) {
+  if (!voiceId || voiceId === 'default') return ensureSpeakerEmbeddings();
+  if (voiceEmbeddingsCache.has(voiceId)) return voiceEmbeddingsCache.get(voiceId);
+  const binPath = path.join(DATA_DIR, `speaker_${voiceId}.bin`);
+  if (fs.existsSync(binPath)) {
+    const buf = fs.readFileSync(binPath);
+    const emb = new Float32Array(new Uint8Array(buf).buffer);
+    voiceEmbeddingsCache.set(voiceId, emb);
+    return emb;
+  }
+  if (voiceId.startsWith('custom_')) {
+    return generateEmbeddingFromCustomVoice(voiceId);
+  }
+  const offset = SPEAKER_OFFSETS[voiceId];
+  if (offset === undefined) return ensureSpeakerEmbeddings();
+  const url = `${DATASET_API}&offset=${offset}&length=${SAMPLES_TO_AVERAGE}`;
+  const resp = await fetch(url);
+  if (!resp.ok) throw new Error('Failed to fetch voice embeddings for ' + voiceId);
+  const data = await resp.json();
+  const avg = new Float32Array(512);
+  let count = 0;
+  for (const item of data.rows) {
+    const match = item.row.filename.match(/cmu_us_(\w+)_arctic/);
+    if (match && match[1] === voiceId) {
+      for (let i = 0; i < 512; i++) avg[i] += item.row.xvector[i];
+      count++;
+    }
+  }
+  if (count === 0) return ensureSpeakerEmbeddings();
+  for (let i = 0; i < 512; i++) avg[i] /= count;
+  if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+  fs.writeFileSync(binPath, Buffer.from(avg.buffer));
+  voiceEmbeddingsCache.set(voiceId, avg);
+  return avg;
+}
+
+async function getSpeakerEmbeddingPipeline() {
+  if (speakerEmbeddingPipeline) return speakerEmbeddingPipeline;
+  if (speakerEmbeddingLoading) {
+    while (speakerEmbeddingLoading) await new Promise(r => setTimeout(r, 100));
+    if (!speakerEmbeddingPipeline) throw new Error('Speaker embedding pipeline failed to load');
+    return speakerEmbeddingPipeline;
+  }
+  speakerEmbeddingLoading = true;
+  try {
+    const { pipeline, env } = await loadTransformers();
+    env.allowRemoteModels = true;
+    speakerEmbeddingPipeline = await pipeline('feature-extraction', 'speechbrain/spkrec-xvectors-voxceleb', {
+      device: 'cpu',
+      dtype: 'fp32',
+    });
+    return speakerEmbeddingPipeline;
+  } catch (err) {
+    speakerEmbeddingPipeline = null;
+    throw new Error('Speaker embedding model load failed: ' + err.message);
+  } finally {
+    speakerEmbeddingLoading = false;
+  }
+}
+
+function findCustomVoiceFile(voiceId) {
+  const baseName = voiceId.replace(/^custom_/, '');
+  for (const dir of [VOICES_DIR, HOME_VOICES_DIR]) {
+    for (const ext of AUDIO_EXTENSIONS) {
+      const candidate = path.join(dir, baseName + ext);
+      if (fs.existsSync(candidate)) return candidate;
+    }
+  }
+  return null;
+}
+
+async function decodeAudioFile(filePath) {
+  const buf = fs.readFileSync(filePath);
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === '.wav') {
+    const decoded = decodeWavToFloat32(buf);
+    return resampleTo16k(decoded.audio, decoded.sampleRate);
+  }
+  const decode = (await import('audio-decode')).default;
+  const audioBuffer = await decode(buf);
+  const mono = audioBuffer.getChannelData(0);
+  return resampleTo16k(mono, audioBuffer.sampleRate);
+}
+
+async function generateEmbeddingFromCustomVoice(voiceId) {
+  const audioFile = findCustomVoiceFile(voiceId);
+  if (!audioFile) {
+    console.error('[VOICES] Custom voice file not found for:', voiceId);
+    return ensureSpeakerEmbeddings();
+  }
+  console.log('[VOICES] Generating embedding from:', audioFile);
+  const audio = await decodeAudioFile(audioFile);
+  if (audio.length < SAMPLE_RATE_STT * 0.5) {
+    throw new Error('Audio too short for embedding extraction (need at least 0.5 seconds)');
+  }
+  const pipe = await getSpeakerEmbeddingPipeline();
+  const output = await pipe(audio, { pooling: 'mean', normalize: true });
+  const embedding = new Float32Array(512);
+  for (let i = 0; i < Math.min(512, output.data.length); i++) {
+    embedding[i] = output.data[i];
+  }
+  if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+  const binPath = path.join(DATA_DIR, `speaker_${voiceId}.bin`);
+  fs.writeFileSync(binPath, Buffer.from(embedding.buffer));
+  voiceEmbeddingsCache.set(voiceId, embedding);
+  console.log('[VOICES] Generated embedding for custom voice:', voiceId);
+  return embedding;
+}
+
 async function getSTT() {
   if (sttPipeline) return sttPipeline;
   if (sttLoading) {
@@ -233,36 +398,38 @@ function cachePut(key, buf) {
   ttsCache.set(key, buf);
 }
 
-async function synthesize(text) {
-  const cached = ttsCache.get(text);
+async function synthesize(text, voiceId) {
+  const cacheKey = (voiceId || 'default') + ':' + text;
+  const cached = ttsCache.get(cacheKey);
   if (cached) {
-    ttsCache.delete(text);
-    ttsCache.set(text, cached);
+    ttsCache.delete(cacheKey);
+    ttsCache.set(cacheKey, cached);
     return cached;
   }
   const tts = await getTTS();
-  const embeddings = await ensureSpeakerEmbeddings();
+  const embeddings = await loadVoiceEmbedding(voiceId);
   const result = await tts(text, { speaker_embeddings: embeddings });
   const wav = encodeWav(result.audio, result.sampling_rate || SAMPLE_RATE_TTS);
-  cachePut(text, wav);
+  cachePut(cacheKey, wav);
   return wav;
 }
 
-async function* synthesizeStream(text) {
+async function* synthesizeStream(text, voiceId) {
   const sentences = splitSentences(text);
   const tts = await getTTS();
-  const embeddings = await ensureSpeakerEmbeddings();
+  const embeddings = await loadVoiceEmbedding(voiceId);
   for (const sentence of sentences) {
-    const cached = ttsCache.get(sentence);
+    const cacheKey = (voiceId || 'default') + ':' + sentence;
+    const cached = ttsCache.get(cacheKey);
     if (cached) {
-      ttsCache.delete(sentence);
-      ttsCache.set(sentence, cached);
+      ttsCache.delete(cacheKey);
+      ttsCache.set(cacheKey, cached);
       yield cached;
       continue;
     }
     const result = await tts(sentence, { speaker_embeddings: embeddings });
     const wav = encodeWav(result.audio, result.sampling_rate || SAMPLE_RATE_TTS);
-    cachePut(sentence, wav);
+    cachePut(cacheKey, wav);
     yield wav;
   }
 }
@@ -276,4 +443,4 @@ function getStatus() {
   };
 }
 
-export { transcribe, synthesize, synthesizeStream, getSTT, getTTS, getStatus };
+export { transcribe, synthesize, synthesizeStream, getSTT, getTTS, getStatus, getVoices };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgui",
-  "version": "1.0.168",
+  "version": "1.0.170",
   "description": "Multi-agent ACP client with real-time communication",
   "type": "module",
   "main": "server.js",
@@ -23,6 +23,7 @@
   "dependencies": {
     "@anthropic-ai/claude-code": "^2.1.37",
     "@huggingface/transformers": "^3.8.1",
+    "audio-decode": "^2.2.3",
     "better-sqlite3": "^12.6.2",
     "busboy": "^1.6.0",
     "express": "^5.2.1",

package/server.js

@@ -535,16 +535,27 @@ const server = http.createServer(async (req, res) => {
       return;
     }
 
+    if (routePath === '/api/voices' && req.method === 'GET') {
+      try {
+        const { getVoices } = await getSpeech();
+        sendJSON(req, res, 200, { ok: true, voices: getVoices() });
+      } catch (err) {
+        sendJSON(req, res, 200, { ok: true, voices: [] });
+      }
+      return;
+    }
+
     if (routePath === '/api/tts' && req.method === 'POST') {
       try {
         const body = await parseBody(req);
         const text = body.text || '';
+        const voiceId = body.voiceId || null;
         if (!text) {
-                    sendJSON(req, res, 400, { error: 'No text provided' });
+          sendJSON(req, res, 400, { error: 'No text provided' });
           return;
         }
         const { synthesize } = await getSpeech();
-        const wavBuffer = await synthesize(text);
+        const wavBuffer = await synthesize(text, voiceId);
         res.writeHead(200, { 'Content-Type': 'audio/wav', 'Content-Length': wavBuffer.length });
         res.end(wavBuffer);
       } catch (err) {
@@ -558,6 +569,7 @@ const server = http.createServer(async (req, res) => {
       try {
         const body = await parseBody(req);
         const text = body.text || '';
+        const voiceId = body.voiceId || null;
         if (!text) {
           sendJSON(req, res, 400, { error: 'No text provided' });
           return;
@@ -569,7 +581,7 @@ const server = http.createServer(async (req, res) => {
           'X-Content-Type': 'audio/wav-stream',
           'Cache-Control': 'no-cache'
         });
-        for await (const wavChunk of synthesizeStream(text)) {
+        for await (const wavChunk of synthesizeStream(text, voiceId)) {
           const lenBuf = Buffer.alloc(4);
           lenBuf.writeUInt32BE(wavChunk.length, 0);
           res.write(lenBuf);
@@ -589,7 +601,7 @@ const server = http.createServer(async (req, res) => {
         const { getStatus } = await getSpeech();
         sendJSON(req, res, 200, getStatus());
       } catch (err) {
-                sendJSON(req, res, 200, { sttReady: false, ttsReady: false, sttLoading: false, ttsLoading: false });
+        sendJSON(req, res, 200, { sttReady: false, ttsReady: false, sttLoading: false, ttsLoading: false });
       }
       return;
     }

package/static/index.html

@@ -1056,6 +1056,28 @@
       border-color: var(--color-error);
     }
 
+    .voice-selector-wrapper {
+      display: flex;
+      align-items: center;
+      gap: 0.25rem;
+    }
+
+    .voice-selector {
+      padding: 0.2rem 0.5rem;
+      border: 1px solid var(--color-border);
+      border-radius: 0.375rem;
+      background: var(--color-bg-secondary);
+      color: var(--color-text-primary);
+      font-size: 0.75rem;
+      cursor: pointer;
+      max-width: 160px;
+    }
+
+    .voice-selector:focus {
+      outline: none;
+      border-color: var(--color-primary);
+    }
+
     .voice-empty {
       text-align: center;
       color: var(--color-text-secondary);
@@ -2146,6 +2168,11 @@
               <input type="checkbox" id="voiceTTSToggle" checked>
               <span>Auto-speak responses</span>
             </label>
+            <div class="voice-selector-wrapper">
+              <select class="voice-selector" id="voiceSelector" title="Select voice">
+                <option value="default">Default</option>
+              </select>
+            </div>
             <button class="voice-stop-btn" id="voiceStopSpeaking" title="Stop speaking">Stop</button>
           </div>
         </div>

package/static/js/voice.js

@@ -14,12 +14,62 @@
   var TARGET_SAMPLE_RATE = 16000;
   var spokenChunks = new Set();
   var isLoadingHistory = false;
+  var selectedVoiceId = localStorage.getItem('voice-selected-id') || 'default';
 
   function init() {
     setupTTSToggle();
     setupUI();
     setupStreamingListener();
     setupAgentSelector();
+    setupVoiceSelector();
+  }
+
+  function setupVoiceSelector() {
+    var selector = document.getElementById('voiceSelector');
+    if (!selector) return;
+    var saved = localStorage.getItem('voice-selected-id');
+    if (saved) selectedVoiceId = saved;
+    fetch(BASE + '/api/voices')
+      .then(function(res) { return res.json(); })
+      .then(function(data) {
+        if (!data.ok || !Array.isArray(data.voices)) return;
+        selector.innerHTML = '';
+        var builtIn = data.voices.filter(function(v) { return !v.isCustom; });
+        var custom = data.voices.filter(function(v) { return v.isCustom; });
+        if (builtIn.length) {
+          var grp1 = document.createElement('optgroup');
+          grp1.label = 'Built-in Voices';
+          builtIn.forEach(function(voice) {
+            var opt = document.createElement('option');
+            opt.value = voice.id;
+            var parts = [];
+            if (voice.gender) parts.push(voice.gender);
+            if (voice.accent) parts.push(voice.accent);
+            opt.textContent = voice.name + (parts.length ? ' (' + parts.join(', ') + ')' : '');
+            grp1.appendChild(opt);
+          });
+          selector.appendChild(grp1);
+        }
+        if (custom.length) {
+          var grp2 = document.createElement('optgroup');
+          grp2.label = 'Custom Voices';
+          custom.forEach(function(voice) {
+            var opt = document.createElement('option');
+            opt.value = voice.id;
+            opt.textContent = voice.name;
+            grp2.appendChild(opt);
+          });
+          selector.appendChild(grp2);
+        }
+        if (saved && selector.querySelector('option[value="' + saved + '"]')) {
+          selector.value = saved;
+        }
+      })
+      .catch(function() {});
+    selector.addEventListener('change', function() {
+      selectedVoiceId = selector.value;
+      localStorage.setItem('voice-selected-id', selectedVoiceId);
+    });
   }
 
   function syncVoiceSelector() {
@@ -286,53 +336,78 @@
     var text = speechQueue.shift();
     audioChunkQueue = [];
     isPlayingChunk = false;
-    fetch(BASE + '/api/tts-stream', {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ text: text })
-    }).then(function(resp) {
-      if (!resp.ok) throw new Error('TTS failed');
-      var reader = resp.body.getReader();
-      var buffer = new Uint8Array(0);
-
-      function concat(a, b) {
-        var c = new Uint8Array(a.length + b.length);
-        c.set(a, 0);
-        c.set(b, a.length);
-        return c;
-      }
+    
+    function tryStreaming() {
+      fetch(BASE + '/api/tts-stream', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ text: text, voiceId: selectedVoiceId })
+      }).then(function(resp) {
+        if (!resp.ok) throw new Error('TTS stream failed');
+        var reader = resp.body.getReader();
+        var buffer = new Uint8Array(0);
 
-      function pump() {
-        return reader.read().then(function(result) {
-          if (result.done) {
-            streamDone = true;
-            if (!isPlayingChunk && audioChunkQueue.length === 0) {
-              isSpeaking = false;
-              processQueue();
+        function concat(a, b) {
+          var c = new Uint8Array(a.length + b.length);
+          c.set(a, 0);
+          c.set(b, a.length);
+          return c;
+        }
+
+        function pump() {
+          return reader.read().then(function(result) {
+            if (result.done) {
+              streamDone = true;
+              if (!isPlayingChunk && audioChunkQueue.length === 0) {
+                isSpeaking = false;
+                processQueue();
+              }
+              return;
             }
-            return;
-          }
-          buffer = concat(buffer, result.value);
-          while (buffer.length >= 4) {
-            var view = new DataView(buffer.buffer, buffer.byteOffset, 4);
-            var chunkLen = view.getUint32(0, false);
-            if (buffer.length < 4 + chunkLen) break;
-            var wavData = buffer.slice(4, 4 + chunkLen);
-            buffer = buffer.slice(4 + chunkLen);
-            var blob = new Blob([wavData], { type: 'audio/wav' });
-            audioChunkQueue.push(blob);
-            if (!isPlayingChunk) playNextChunk();
-          }
-          return pump();
-        });
-      }
+            buffer = concat(buffer, result.value);
+            while (buffer.length >= 4) {
+              var view = new DataView(buffer.buffer, buffer.byteOffset, 4);
+              var chunkLen = view.getUint32(0, false);
+              if (buffer.length < 4 + chunkLen) break;
+              var wavData = buffer.slice(4, 4 + chunkLen);
+              buffer = buffer.slice(4 + chunkLen);
+              var blob = new Blob([wavData], { type: 'audio/wav' });
+              audioChunkQueue.push(blob);
+              if (!isPlayingChunk) playNextChunk();
+            }
+            return pump();
+          });
+        }
 
-      return pump();
-    }).catch(function() {
-      streamDone = true;
-      isSpeaking = false;
-      processQueue();
-    });
+        return pump();
+      }).catch(function() {
+        tryNonStreaming(text);
+      });
+    }
+    
+    function tryNonStreaming(txt) {
+      fetch(BASE + '/api/tts', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ text: txt, voiceId: selectedVoiceId })
+      }).then(function(resp) {
+        if (!resp.ok) throw new Error('TTS failed');
+        return resp.arrayBuffer();
+      }).then(function(buf) {
+        var blob = new Blob([buf], { type: 'audio/wav' });
+        audioChunkQueue.push(blob);
+        if (!isPlayingChunk) playNextChunk();
+        streamDone = true;
+        isSpeaking = false;
+        processQueue();
+      }).catch(function() {
+        streamDone = true;
+        isSpeaking = false;
+        processQueue();
+      });
+    }
+    
+    tryStreaming();
   }
 
   function stopSpeaking() {