metaverse-avatar 0.1.0

package/speech.js

@@ -0,0 +1,130 @@
+// Audio-clip-driven lip sync (the playback counterpart to VoiceMouth).
+//
+// Plays a speech clip (xAI TTS audio relayed by the MCP server) out loud
+// through Web Audio. When a viseme timing file accompanies the clip, the
+// mouth is shaped per-viseme from the precise playback time; otherwise it
+// falls back to the microphone-style AnalyserNode RMS → jaw mapping. One
+// clip plays at a time; starting a new one cancels the previous.
+
+import { buildVisemeTimeline, sampleViseme } from './visemes.js';
+
+export class SpeechMouth {
+  constructor(avatar) {
+    this.avatar = avatar;
+    this.active = false;
+    this.level = 0; // smoothed jaw amount 0..1 (RMS fallback)
+
+    // RMS-fallback envelope (matches VoiceMouth).
+    this.gain = 17;
+    this.maxOpen = 1;
+    this.gate = 0.015;
+    this.attack = 35;
+    this.release = 12;
+
+    this._ctx = null;      // reused across clips (browsers cap AudioContexts)
+    this._audio = null;
+    this._src = null;
+    this._analyser = null;
+    this._data = null;
+
+    this._timeline = null;                       // viseme segments, or null
+    this._m = { open: 0, round: 0, wide: 0 };    // smoothed viseme controls
+    this._out = { open: 0, round: 0, wide: 0 };  // scratch passed to setMouth
+  }
+
+  // Play the clip at `url` and lip-sync to it. If `visemeUrl` is given, fetch
+  // the viseme timing file and drive per-viseme mouth shapes; otherwise use
+  // the amplitude fallback. Resolves once playback begins, returning
+  // { duration } (seconds, or null if unknown).
+  async play(url, visemeUrl) {
+    this.stop();
+    if (!this._ctx) this._ctx = new (window.AudioContext || window.webkitAudioContext)();
+    await this._ctx.resume(); // may stay suspended without a prior user gesture
+
+    // Viseme timing is fetched in parallel with audio setup; same-origin.
+    const timelinePromise = visemeUrl
+      ? fetch(visemeUrl).then((r) => r.json()).then(buildVisemeTimeline).catch(() => null)
+      : Promise.resolve(null);
+
+    const audio = new Audio();
+    audio.crossOrigin = 'anonymous';
+    audio.src = url;
+
+    const src = this._ctx.createMediaElementSource(audio);
+    const analyser = this._ctx.createAnalyser();
+    analyser.fftSize = 1024;
+    analyser.smoothingTimeConstant = 0.4;
+    src.connect(analyser);
+    analyser.connect(this._ctx.destination); // route to speakers + the meter
+    this._audio = audio;
+    this._src = src;
+    this._analyser = analyser;
+    this._data = new Uint8Array(analyser.fftSize);
+
+    await new Promise((resolve, reject) => {
+      audio.addEventListener('loadedmetadata', resolve, { once: true });
+      audio.addEventListener('error', () => reject(new Error(`failed to load audio: ${url}`)), { once: true });
+    });
+    audio.addEventListener('ended', () => this.stop(), { once: true });
+
+    this._timeline = await timelinePromise;
+    this._m.open = this._m.round = this._m.wide = 0;
+    this.active = true;
+    await audio.play();
+    return { duration: Number.isFinite(audio.duration) ? audio.duration : null, visemes: !!this._timeline };
+  }
+
+  stop() {
+    if (this._audio) {
+      this._audio.pause();
+      this._audio.removeAttribute('src');
+    }
+    this._src?.disconnect();
+    this._analyser?.disconnect();
+    this._audio = null;
+    this._src = null;
+    this._analyser = null;
+    this._data = null;
+    this._timeline = null;
+    this._m.open = this._m.round = this._m.wide = 0;
+    this.active = false;
+    this.level = 0;
+    this.avatar.setMouth?.({ open: 0, round: 0, wide: 0 });
+    this.avatar.setMouthOpen(0);
+  }
+
+  update(dt) {
+    if (!this.active) return;
+
+    // Viseme mode: shape the mouth from the precise playback time.
+    if (this._timeline) {
+      const t = this._audio ? this._audio.currentTime : 0;
+      const target = sampleViseme(this._timeline, t);
+      const k = 1 - Math.exp(-22 * Math.min(dt, 0.1)); // smooth coarticulation
+      this._m.open += (target.open - this._m.open) * k;
+      this._m.round += (target.round - this._m.round) * k;
+      this._m.wide += (target.wide - this._m.wide) * k;
+      // "Max open" caps the jaw; lip round/spread shapes stay full so visemes
+      // remain legible even at a low cap.
+      this._out.open = this._m.open * this.maxOpen;
+      this._out.round = this._m.round;
+      this._out.wide = this._m.wide;
+      this.avatar.setMouth(this._out);
+      return;
+    }
+
+    // Fallback: amplitude-driven jaw.
+    if (!this._analyser) return;
+    this._analyser.getByteTimeDomainData(this._data);
+    let sum = 0;
+    for (let i = 0; i < this._data.length; i++) {
+      const v = (this._data[i] - 128) / 128;
+      sum += v * v;
+    }
+    const rms = Math.sqrt(sum / this._data.length);
+    const target = Math.min(1, Math.max(0, (rms - this.gate) * this.gain));
+    const k = target > this.level ? this.attack : this.release;
+    this.level += (target - this.level) * (1 - Math.exp(-k * Math.min(dt, 0.1)));
+    this.avatar.setMouthOpen(this.level * this.maxOpen);
+  }
+}

package/visemes.js

@@ -0,0 +1,66 @@
+// Viseme model for speech lip-sync.
+//
+// xAI TTS returns per-character timing (graph_chars + graph_times). We map
+// each grapheme to one of a small set of visemes, each a target for three
+// mouth controls (consumed by RuthAvatar.setMouth):
+//   open  — jaw drop (vowels)
+//   round — lip pucker / protrude (o, u, w)
+//   wide  — lip spread (e, i)
+//
+// English spelling → phoneme is irregular, so this is a heuristic grapheme
+// mapping, not a phonemizer — but with correct timing and real closures
+// (m/b/p shut the lips) it reads far better than amplitude-only jaw motion.
+
+export const VISEMES = {
+  sil: { open: 0.0,  round: 0.0,  wide: 0.0 },  // silence / rest (closed)
+  PP:  { open: 0.0,  round: 0.0,  wide: 0.05 }, // p b m — lips closed
+  FF:  { open: 0.12, round: 0.0,  wide: 0.25 }, // f v
+  TH:  { open: 0.18, round: 0.0,  wide: 0.20 }, // th
+  DD:  { open: 0.25, round: 0.0,  wide: 0.35 }, // generic consonant (d t n s z l c k g …)
+  RR:  { open: 0.22, round: 0.35, wide: 0.0 },  // r
+  AA:  { open: 0.90, round: 0.0,  wide: 0.20 }, // a
+  E:   { open: 0.45, round: 0.0,  wide: 0.60 }, // e
+  I:   { open: 0.28, round: 0.0,  wide: 0.80 }, // i y
+  O:   { open: 0.55, round: 0.70, wide: 0.0 },  // o
+  U:   { open: 0.32, round: 0.95, wide: 0.0 },  // u w
+};
+
+const CHAR_VISEME = {
+  a: 'AA', e: 'E', i: 'I', o: 'O', u: 'U', y: 'I', w: 'U',
+  m: 'PP', b: 'PP', p: 'PP',
+  f: 'FF', v: 'FF',
+  r: 'RR',
+  // every other consonant collapses to a generic open-consonant shape
+  d: 'DD', t: 'DD', n: 'DD', s: 'DD', z: 'DD', l: 'DD', c: 'DD',
+  k: 'DD', g: 'DD', h: 'DD', j: 'DD', q: 'DD', x: 'DD',
+};
+
+export function charToViseme(ch) {
+  return CHAR_VISEME[(ch ?? '').toLowerCase()] ?? 'sil'; // spaces / punctuation / digits
+}
+
+// Build a time-ordered list of { start, end, viseme } from xAI's
+// audio_timestamps ({ graph_chars, graph_times: [[start, end], …] }).
+export function buildVisemeTimeline(timestamps) {
+  const chars = timestamps?.graph_chars ?? [];
+  const times = timestamps?.graph_times ?? [];
+  const segs = [];
+  for (let i = 0; i < chars.length; i++) {
+    const t = times[i];
+    if (!t || typeof t[0] !== 'number') continue;
+    segs.push({ start: t[0], end: typeof t[1] === 'number' ? t[1] : t[0], viseme: charToViseme(chars[i]) });
+  }
+  return segs;
+}
+
+// Target mouth controls for playback time `t` (seconds). Holds the most
+// recently started viseme, relaxing to neutral during pauses between words.
+export function sampleViseme(timeline, t) {
+  if (!timeline || timeline.length === 0) return VISEMES.sil;
+  let seg = null;
+  for (const s of timeline) {
+    if (s.start <= t) seg = s; else break;
+  }
+  if (!seg || t > seg.end + 0.12) return VISEMES.sil;
+  return VISEMES[seg.viseme] ?? VISEMES.sil;
+}

package/voice.js

@@ -0,0 +1,75 @@
+// Microphone-driven lip sync.
+//
+// Taps the mic through a Web Audio AnalyserNode, measures the time-domain
+// RMS each frame, gates out background noise, and maps the result to a jaw
+// open amount with a fast attack / slower release envelope so the mouth
+// tracks speech without chattering. The smoothed level is pushed to
+// RuthAvatar.setMouthOpen every frame.
+
+export class VoiceMouth {
+  constructor(avatar) {
+    this.avatar = avatar;
+    this.active = false;
+    this.level = 0; // smoothed jaw amount 0..1 (also the meter value)
+
+    this.gain = 17;     // rms → open scale (Sensitivity slider)
+    this.maxOpen = 1;   // jaw cap 0..1 (Max open slider)
+    this.gate = 0.015;  // noise floor below which the mouth stays shut
+    this.attack = 35;   // 1/s — open quickly
+    this.release = 12;  // 1/s — close more slowly
+
+    this._stream = null;
+    this._ctx = null;
+    this._analyser = null;
+    this._data = null;
+  }
+
+  async start() {
+    if (this.active) return;
+    if (!navigator.mediaDevices?.getUserMedia) {
+      throw new Error('microphone not available in this browser');
+    }
+    this._stream = await navigator.mediaDevices.getUserMedia({
+      audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: false },
+    });
+    this._ctx = new (window.AudioContext || window.webkitAudioContext)();
+    await this._ctx.resume();
+    const src = this._ctx.createMediaStreamSource(this._stream);
+    this._analyser = this._ctx.createAnalyser();
+    this._analyser.fftSize = 1024;
+    this._analyser.smoothingTimeConstant = 0.4;
+    src.connect(this._analyser);
+    this._data = new Uint8Array(this._analyser.fftSize);
+    this.active = true;
+  }
+
+  stop() {
+    if (this._stream) for (const t of this._stream.getTracks()) t.stop();
+    if (this._ctx) this._ctx.close();
+    this._stream = null;
+    this._ctx = null;
+    this._analyser = null;
+    this._data = null;
+    this.active = false;
+    this.level = 0;
+    this.avatar.setMouthOpen(0);
+  }
+
+  update(dt) {
+    if (!this.active || !this._analyser) return;
+    this._analyser.getByteTimeDomainData(this._data);
+
+    let sum = 0;
+    for (let i = 0; i < this._data.length; i++) {
+      const v = (this._data[i] - 128) / 128;
+      sum += v * v;
+    }
+    const rms = Math.sqrt(sum / this._data.length);
+
+    const target = Math.min(1, Math.max(0, (rms - this.gate) * this.gain));
+    const k = target > this.level ? this.attack : this.release;
+    this.level += (target - this.level) * (1 - Math.exp(-k * Math.min(dt, 0.1)));
+
+    this.avatar.setMouthOpen(this.level * this.maxOpen);
+  }
+}