agentgui 1.0.169 → 1.0.171

package/lib/speech.js

@@ -1,5 +1,6 @@
 import { createRequire } from 'module';
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
@@ -7,16 +8,22 @@ const require = createRequire(import.meta.url);
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT = path.dirname(__dirname);
 const DATA_DIR = path.join(ROOT, 'data');
+const AUDIO_EXTENSIONS = ['.mp3', '.wav', '.ogg', '.flac', '.m4a'];
 
-const SPEAKER_EMBEDDINGS_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
-const SPEAKER_EMBEDDINGS_PATH = path.join(DATA_DIR, 'speaker_embeddings.bin');
-const SAMPLE_RATE_TTS = 16000;
-const SAMPLE_RATE_STT = 16000;
-const MIN_WAV_SIZE = 44;
-const DATASET_API = 'https://datasets-server.huggingface.co/rows?dataset=Matthijs%2Fcmu-arctic-xvectors&config=default&split=validation';
-const SAMPLES_TO_AVERAGE = 10;
+function getVoiceDirs() {
+  const dirs = [];
+  const seen = new Set();
+  const add = (d) => { const r = path.resolve(d); if (!seen.has(r)) { seen.add(r); dirs.push(r); } };
+  const startupCwd = process.env.STARTUP_CWD || process.cwd();
+  add(path.join(startupCwd, 'voices'));
+  add(path.join(ROOT, 'voices'));
+  add(path.join(os.homedir(), 'voices'));
+  return dirs;
+}
 
-const VOICE_CATALOG = [
+const MIN_WAV_SIZE = 1000;
+
+const BASE_VOICES = [
   { id: 'default', name: 'Default', gender: 'male', accent: 'US' },
   { id: 'bdl', name: 'BDL', gender: 'male', accent: 'US' },
   { id: 'slt', name: 'SLT', gender: 'female', accent: 'US' },
@@ -27,15 +34,88 @@ const VOICE_CATALOG = [
   { id: 'ksp', name: 'KSP', gender: 'male', accent: 'Indian' },
 ];
 
+async function convertToWav(filePath) {
+  const wavPath = filePath.replace(/\.[^.]+$/, '.wav');
+  if (fs.existsSync(wavPath)) return wavPath;
+  try {
+    console.log('[VOICES] Converting to WAV:', filePath);
+    const audio = await decodeAudioFile(filePath);
+    const wav = encodeWav(audio, SAMPLE_RATE_STT);
+    fs.writeFileSync(wavPath, wav);
+    console.log('[VOICES] Converted:', path.basename(wavPath));
+    return wavPath;
+  } catch (err) {
+    console.error('[VOICES] Conversion failed for', filePath + ':', err.message);
+    return null;
+  }
+}
+
+const pendingConversions = new Map();
+
+function scanVoiceDir(dir) {
+  const voices = [];
+  try {
+    if (!fs.existsSync(dir)) return voices;
+    for (const file of fs.readdirSync(dir)) {
+      const ext = path.extname(file).toLowerCase();
+      if (!AUDIO_EXTENSIONS.includes(ext)) continue;
+      const baseName = path.basename(file, ext);
+      if (ext !== '.wav') {
+        const wavFile = baseName + '.wav';
+        if (fs.existsSync(path.join(dir, wavFile))) continue;
+        const fullPath = path.join(dir, file);
+        if (!pendingConversions.has(fullPath)) {
+          pendingConversions.set(fullPath, convertToWav(fullPath).then(result => {
+            pendingConversions.delete(fullPath);
+            return result;
+          }));
+        }
+        continue;
+      }
+      const id = 'custom_' + baseName.replace(/[^a-zA-Z0-9_-]/g, '_');
+      const name = baseName.replace(/_/g, ' ');
+      voices.push({ id, name, gender: 'custom', accent: 'custom', isCustom: true, sourceDir: dir });
+    }
+  } catch (err) {
+    console.error('[VOICES] Error scanning', dir + ':', err.message);
+  }
+  return voices;
+}
+
+function loadCustomVoices() {
+  const seen = new Set();
+  const voices = [];
+  for (const dir of getVoiceDirs()) {
+    for (const v of scanVoiceDir(dir)) {
+      if (seen.has(v.id)) continue;
+      seen.add(v.id);
+      voices.push(v);
+    }
+  }
+  return voices;
+}
+
+function getVoices() {
+  return [...BASE_VOICES, ...loadCustomVoices()];
+}
+
 const SPEAKER_OFFSETS = { awb: 0, bdl: 1200, clb: 2300, jmk: 3500, ksp: 4700, rms: 5900, slt: 7100 };
+const SPEAKER_EMBEDDINGS_URL = 'https://huggingface.co/datasets/Xenova/speaker_embeddings/resolve/main/spkrec-xvectors-voxceleb.hf';
+const SPEAKER_EMBEDDINGS_PATH = path.join(DATA_DIR, 'speaker_embeddings.bin');
+const DATASET_API = 'https://datasets-server.huggingface.co/rows?dataset=Xenova%2Fspeaker_embeddings&config=default&split=train';
+const SAMPLES_TO_AVERAGE = 30;
 
 let transformersModule = null;
 let sttPipeline = null;
 let ttsPipeline = null;
 let speakerEmbeddings = null;
+let speakerEmbeddingPipeline = null;
 let sttLoading = false;
 let ttsLoading = false;
+let speakerEmbeddingLoading = false;
 const voiceEmbeddingsCache = new Map();
+const SAMPLE_RATE_STT = 16000;
+const SAMPLE_RATE_TTS = 16000;
 
 const TTS_CACHE_MAX = 100;
 const ttsCache = new Map();
@@ -78,6 +158,9 @@ async function loadVoiceEmbedding(voiceId) {
     voiceEmbeddingsCache.set(voiceId, emb);
     return emb;
   }
+  if (voiceId.startsWith('custom_')) {
+    return generateEmbeddingFromCustomVoice(voiceId);
+  }
   const offset = SPEAKER_OFFSETS[voiceId];
   if (offset === undefined) return ensureSpeakerEmbeddings();
   const url = `${DATASET_API}&offset=${offset}&length=${SAMPLES_TO_AVERAGE}`;
@@ -101,8 +184,77 @@ async function loadVoiceEmbedding(voiceId) {
   return avg;
 }
 
-function getVoices() {
-  return VOICE_CATALOG;
+async function getSpeakerEmbeddingPipeline() {
+  if (speakerEmbeddingPipeline) return speakerEmbeddingPipeline;
+  if (speakerEmbeddingLoading) {
+    while (speakerEmbeddingLoading) await new Promise(r => setTimeout(r, 100));
+    if (!speakerEmbeddingPipeline) throw new Error('Speaker embedding pipeline failed to load');
+    return speakerEmbeddingPipeline;
+  }
+  speakerEmbeddingLoading = true;
+  try {
+    const { pipeline, env } = await loadTransformers();
+    env.allowRemoteModels = true;
+    speakerEmbeddingPipeline = await pipeline('feature-extraction', 'speechbrain/spkrec-xvectors-voxceleb', {
+      device: 'cpu',
+      dtype: 'fp32',
+    });
+    return speakerEmbeddingPipeline;
+  } catch (err) {
+    speakerEmbeddingPipeline = null;
+    throw new Error('Speaker embedding model load failed: ' + err.message);
+  } finally {
+    speakerEmbeddingLoading = false;
+  }
+}
+
+function findCustomVoiceFile(voiceId) {
+  const baseName = voiceId.replace(/^custom_/, '');
+  for (const dir of getVoiceDirs()) {
+    for (const ext of AUDIO_EXTENSIONS) {
+      const candidate = path.join(dir, baseName + ext);
+      if (fs.existsSync(candidate)) return candidate;
+    }
+  }
+  return null;
+}
+
+async function decodeAudioFile(filePath) {
+  const buf = fs.readFileSync(filePath);
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === '.wav') {
+    const decoded = decodeWavToFloat32(buf);
+    return resampleTo16k(decoded.audio, decoded.sampleRate);
+  }
+  const decode = (await import('audio-decode')).default;
+  const audioBuffer = await decode(buf);
+  const mono = audioBuffer.getChannelData(0);
+  return resampleTo16k(mono, audioBuffer.sampleRate);
+}
+
+async function generateEmbeddingFromCustomVoice(voiceId) {
+  const audioFile = findCustomVoiceFile(voiceId);
+  if (!audioFile) {
+    console.error('[VOICES] Custom voice file not found for:', voiceId);
+    return ensureSpeakerEmbeddings();
+  }
+  console.log('[VOICES] Generating embedding from:', audioFile);
+  const audio = await decodeAudioFile(audioFile);
+  if (audio.length < SAMPLE_RATE_STT * 0.5) {
+    throw new Error('Audio too short for embedding extraction (need at least 0.5 seconds)');
+  }
+  const pipe = await getSpeakerEmbeddingPipeline();
+  const output = await pipe(audio, { pooling: 'mean', normalize: true });
+  const embedding = new Float32Array(512);
+  for (let i = 0; i < Math.min(512, output.data.length); i++) {
+    embedding[i] = output.data[i];
+  }
+  if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+  const binPath = path.join(DATA_DIR, `speaker_${voiceId}.bin`);
+  fs.writeFileSync(binPath, Buffer.from(embedding.buffer));
+  voiceEmbeddingsCache.set(voiceId, embedding);
+  console.log('[VOICES] Generated embedding for custom voice:', voiceId);
+  return embedding;
 }
 
 async function getSTT() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgui",
-  "version": "1.0.169",
+  "version": "1.0.171",
   "description": "Multi-agent ACP client with real-time communication",
   "type": "module",
   "main": "server.js",
@@ -23,6 +23,7 @@
   "dependencies": {
     "@anthropic-ai/claude-code": "^2.1.37",
     "@huggingface/transformers": "^3.8.1",
+    "audio-decode": "^2.2.3",
     "better-sqlite3": "^12.6.2",
     "busboy": "^1.6.0",
     "express": "^5.2.1",

package/static/js/voice.js

@@ -34,19 +34,33 @@
       .then(function(data) {
         if (!data.ok || !Array.isArray(data.voices)) return;
         selector.innerHTML = '';
-        data.voices.forEach(function(voice) {
-          var opt = document.createElement('option');
-          opt.value = voice.id;
-          var label = voice.name;
-          if (voice.gender || voice.accent) {
+        var builtIn = data.voices.filter(function(v) { return !v.isCustom; });
+        var custom = data.voices.filter(function(v) { return v.isCustom; });
+        if (builtIn.length) {
+          var grp1 = document.createElement('optgroup');
+          grp1.label = 'Built-in Voices';
+          builtIn.forEach(function(voice) {
+            var opt = document.createElement('option');
+            opt.value = voice.id;
             var parts = [];
             if (voice.gender) parts.push(voice.gender);
             if (voice.accent) parts.push(voice.accent);
-            label += ' (' + parts.join(', ') + ')';
-          }
-          opt.textContent = label;
-          selector.appendChild(opt);
-        });
+            opt.textContent = voice.name + (parts.length ? ' (' + parts.join(', ') + ')' : '');
+            grp1.appendChild(opt);
+          });
+          selector.appendChild(grp1);
+        }
+        if (custom.length) {
+          var grp2 = document.createElement('optgroup');
+          grp2.label = 'Custom Voices';
+          custom.forEach(function(voice) {
+            var opt = document.createElement('option');
+            opt.value = voice.id;
+            opt.textContent = voice.name;
+            grp2.appendChild(opt);
+          });
+          selector.appendChild(grp2);
+        }
         if (saved && selector.querySelector('option[value="' + saved + '"]')) {
           selector.value = saved;
         }
@@ -322,53 +336,78 @@
     var text = speechQueue.shift();
     audioChunkQueue = [];
     isPlayingChunk = false;
-    fetch(BASE + '/api/tts-stream', {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ text: text, voiceId: selectedVoiceId })
-    }).then(function(resp) {
-      if (!resp.ok) throw new Error('TTS failed');
-      var reader = resp.body.getReader();
-      var buffer = new Uint8Array(0);
-
-      function concat(a, b) {
-        var c = new Uint8Array(a.length + b.length);
-        c.set(a, 0);
-        c.set(b, a.length);
-        return c;
-      }
+    
+    function tryStreaming() {
+      fetch(BASE + '/api/tts-stream', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ text: text, voiceId: selectedVoiceId })
+      }).then(function(resp) {
+        if (!resp.ok) throw new Error('TTS stream failed');
+        var reader = resp.body.getReader();
+        var buffer = new Uint8Array(0);
+
+        function concat(a, b) {
+          var c = new Uint8Array(a.length + b.length);
+          c.set(a, 0);
+          c.set(b, a.length);
+          return c;
+        }
 
-      function pump() {
-        return reader.read().then(function(result) {
-          if (result.done) {
-            streamDone = true;
-            if (!isPlayingChunk && audioChunkQueue.length === 0) {
-              isSpeaking = false;
-              processQueue();
+        function pump() {
+          return reader.read().then(function(result) {
+            if (result.done) {
+              streamDone = true;
+              if (!isPlayingChunk && audioChunkQueue.length === 0) {
+                isSpeaking = false;
+                processQueue();
+              }
+              return;
             }
-            return;
-          }
-          buffer = concat(buffer, result.value);
-          while (buffer.length >= 4) {
-            var view = new DataView(buffer.buffer, buffer.byteOffset, 4);
-            var chunkLen = view.getUint32(0, false);
-            if (buffer.length < 4 + chunkLen) break;
-            var wavData = buffer.slice(4, 4 + chunkLen);
-            buffer = buffer.slice(4 + chunkLen);
-            var blob = new Blob([wavData], { type: 'audio/wav' });
-            audioChunkQueue.push(blob);
-            if (!isPlayingChunk) playNextChunk();
-          }
-          return pump();
-        });
-      }
+            buffer = concat(buffer, result.value);
+            while (buffer.length >= 4) {
+              var view = new DataView(buffer.buffer, buffer.byteOffset, 4);
+              var chunkLen = view.getUint32(0, false);
+              if (buffer.length < 4 + chunkLen) break;
+              var wavData = buffer.slice(4, 4 + chunkLen);
+              buffer = buffer.slice(4 + chunkLen);
+              var blob = new Blob([wavData], { type: 'audio/wav' });
+              audioChunkQueue.push(blob);
+              if (!isPlayingChunk) playNextChunk();
+            }
+            return pump();
+          });
+        }
 
-      return pump();
-    }).catch(function() {
-      streamDone = true;
-      isSpeaking = false;
-      processQueue();
-    });
+        return pump();
+      }).catch(function() {
+        tryNonStreaming(text);
+      });
+    }
+    
+    function tryNonStreaming(txt) {
+      fetch(BASE + '/api/tts', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ text: txt, voiceId: selectedVoiceId })
+      }).then(function(resp) {
+        if (!resp.ok) throw new Error('TTS failed');
+        return resp.arrayBuffer();
+      }).then(function(buf) {
+        var blob = new Blob([buf], { type: 'audio/wav' });
+        audioChunkQueue.push(blob);
+        if (!isPlayingChunk) playNextChunk();
+        streamDone = true;
+        isSpeaking = false;
+        processQueue();
+      }).catch(function() {
+        streamDone = true;
+        isSpeaking = false;
+        processQueue();
+      });
+    }
+    
+    tryStreaming();
   }
 
   function stopSpeaking() {