@todoforai/voiceloop 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.14",
3
+ "version": "0.1.16",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.d.ts CHANGED
@@ -101,7 +101,29 @@ export interface VoiceTTS {
101
101
  setOnProgress?(fn: TtsProgress | null): void;
102
102
  }
103
103
 
104
+ /** Host-supplied mic capture (React Native, Electron, tests) — replaces getUserMedia + AudioWorklet +
105
+ * Silero. Frames: 16 kHz mono Float32, ~256 ms (4096 samples). The bootstrap energy-VAD runs for
106
+ * the whole session; pair with a continuous STT (Deepgram Flux) for server-side end-of-turn. */
107
+ export interface AudioIO {
108
+ startCapture(onFrame: (f32: Float32Array) => void): Promise<{ stop(): void; setMuted?(muted: boolean): void }>;
109
+ }
110
+ /** Minimal clip surface StreamingTTS drives: position for the spoken cursor, pause() for barge-in,
111
+ * onended (natural end) / onpause (stopped) callbacks. */
112
+ export interface PlaybackClip {
113
+ readonly currentTime: number; readonly duration: number; paused: boolean; ended: boolean;
114
+ onended: (() => void) | null; onpause: (() => void) | null;
115
+ pause(): void;
116
+ }
117
+ /** Host-supplied TTS playout — replaces the shared Web Audio context + AEC loopback sink. */
118
+ export interface PlaybackIO {
119
+ load(blob: Blob): Promise<{ duration: number; start(offsetSec: number): PlaybackClip }>;
120
+ }
121
+ /** Route ALL StreamingTTS playback through `io` (process-wide); null restores Web Audio. */
122
+ export function setPlaybackIO(io: PlaybackIO | null): void;
123
+
104
124
  export interface VoiceAgentOptions {
125
+ /** Native mic capture; omit for the browser pipeline. See AudioIO. */
126
+ audioIO?: AudioIO;
105
127
  sysmsg?: string;
106
128
  /** Persona/system prompt prepended to the conversation context. Defaults to the built-in
107
129
  * voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
@@ -109,8 +131,8 @@ export interface VoiceAgentOptions {
109
131
  /** LLM model id sent to the backend `/llm` route; empty → the backend's VOICE_MODEL default. */
110
132
  model?: string;
111
133
  llm?: LLM;
112
- /** Transport seam for the built-in LLM adapter; defaults to global fetch. For tests and hosts
113
- * that must wrap every request (auth refresh, tracing). Unused when you pass your own `llm`. */
134
+ /** Transport seam for the built-in LLM adapter AND the STT usage reports; defaults to global
135
+ * fetch. For tests and hosts that must wrap every request (auth refresh, tracing). */
114
136
  fetchFn?: typeof fetch;
115
137
  apiKey?: string;
116
138
  /** Full endpoint URLs, not a base: voiceloop composes no routes, so a host can point the LLM and
package/src/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  export {
2
2
  VoiceAgent, StreamingTTS, PiperTTS,
3
- unlockAudio, warmVad,
3
+ unlockAudio, warmVad, setPlaybackIO,
4
4
  isSelfEcho, novelChars, toolResultText,
5
5
  VOICE_SYSMSG, LANG_NAMES,
6
6
  } from './voice-agent.js';
package/src/stt.js CHANGED
@@ -48,7 +48,7 @@ const newSocket = (url, protocols) => {
48
48
  };
49
49
 
50
50
  // Common opts shape (provided by the VoiceAgent):
51
- // { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl,
51
+ // { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl, fetchFn (usage POSTs),
52
52
  // onPartial, onFinal, onError, onClose, isClosed }
53
53
  //
54
54
  // AUTH (cloud providers): the raw provider key must stay server-side. Either
@@ -107,12 +107,12 @@ async function mintSttToken(url, apiKey, onFatal, getToken) {
107
107
  // Accumulate streamed samples and flush them to /stt/usage — inline once USAGE_FLUSH_SECONDS is
108
108
  // crossed, and again whenever the caller flushes on close (billing is trust-but-verify: the client
109
109
  // reports what it streamed since the backend can't meter a direct browser↔provider socket).
110
- function makeUsageReporter(sttUsageUrl, apiKey, provider, extra = {}) {
110
+ function makeUsageReporter(sttUsageUrl, apiKey, provider, extra = {}, fetchFn = fetch) {
111
111
  let samples = 0;
112
112
  const flush = () => {
113
113
  const seconds = samples / 16000; samples = 0;
114
114
  if (seconds <= 0 || !sttUsageUrl) return;
115
- fetch(sttUsageUrl, {
115
+ fetchFn(sttUsageUrl, {
116
116
  method: 'POST', keepalive: true,
117
117
  headers: { 'Content-Type': 'application/json', 'X-API-Key': apiKey },
118
118
  body: JSON.stringify({ seconds, provider, ...extra }),
@@ -133,7 +133,7 @@ export function makeElevenLabsSTT(opts) {
133
133
  const sttUrl = opts.sttUrl || 'wss://api.elevenlabs.io/v1/speech-to-text/realtime';
134
134
  const sttModel = opts.sttModel || 'scribe_v2_realtime';
135
135
  let ws = null, opening = false, outbox = [], sentSinceCommit = false, sttStart = 0;
136
- const usage = makeUsageReporter(sttUsageUrl, apiKey, 'elevenlabs');
136
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'elevenlabs', {}, opts.fetchFn);
137
137
 
138
138
  const sendAudio = (i16) => {
139
139
  if (!ws || ws.readyState !== 1) { outbox.push(i16); return; }
@@ -277,7 +277,7 @@ export function makeSpeechmaticsSTT(opts) {
277
277
  flushFinal(true);
278
278
  }, ms); };
279
279
 
280
- const usage = makeUsageReporter(sttUsageUrl, apiKey, 'speechmatics', { model: sttModel || 'standard' });
280
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'speechmatics', { model: sttModel || 'standard' }, opts.fetchFn);
281
281
 
282
282
  const sendAudio = (i16) => {
283
283
  if (!ws || ws.readyState !== 1 || !started) { outbox.push(i16); return; }
@@ -485,7 +485,7 @@ export function makeDeepgramSTT(opts) {
485
485
  // TurnInfo.transcript is cumulative for the CURRENT turn (Flux restarts it on each new turn_index),
486
486
  // so `turnText` is simply the latest transcript — no committed/tail split needed.
487
487
  let turnText = '', audioSinceFinal = false, awaitingFinal = false, safetyTimer = null;
488
- const usage = makeUsageReporter(sttUsageUrl, apiKey, 'deepgram', { model });
488
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'deepgram', { model }, opts.fetchFn);
489
489
 
490
490
  // Pre-open buffer is a bounded PREROLL (~1s), not a backlog: Flux is a real-time model — burst-
491
491
  // sending seconds of buffered audio after a slow token fetch/handshake makes its whole timeline lag
@@ -172,7 +172,13 @@ export class VoiceAgent {
172
172
  tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
173
173
  sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
174
174
  tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
175
- maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, onEvent = () => {} } = {}) {
175
+ maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, audioIO = null, onEvent = () => {} } = {}) {
176
+ // Host-supplied mic capture (React Native, Electron main, tests): `startCapture(onFrame)` delivers
177
+ // 16 kHz mono Float32 frames (~256 ms) and resolves a handle with stop()/setMuted(). Replaces the
178
+ // whole getUserMedia + AudioWorklet + Silero path — the bootstrap energy-VAD then runs for the
179
+ // session (no ONNX/WASM on native), which is fine with a continuous STT (Deepgram Flux) that
180
+ // does end-of-turn itself; VAD only drives state events + barge-in reset there.
181
+ this.audioIO = audioIO; this._capture = null;
176
182
  // Each agent gets its own PiperTTS (it owns an AudioContext + playing node) — a shared
177
183
  // module-level default would make two agents fight over one audio output. `speed` scales
178
184
  // playback rate (1.2 → 20% faster) on the default TTS; ignored if a custom `tts` is passed.
@@ -196,7 +202,7 @@ export class VoiceAgent {
196
202
  // `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
197
203
  // whose credential differs per provider never has to re-derive the webspeech downgrade
198
204
  // (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
199
- sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, eotThreshold: sttEotThreshold,
205
+ sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, fetchFn, eotThreshold: sttEotThreshold,
200
206
  onPartial: (text, ms, committed = '') => {
201
207
  const interim = `${committed} ${text}`.trim();
202
208
  this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
@@ -356,6 +362,7 @@ export class VoiceAgent {
356
362
  setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
357
363
  return;
358
364
  }
365
+ if (this.audioIO) return this._startNative();
359
366
  // WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
360
367
  // mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
361
368
  // permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
@@ -463,6 +470,27 @@ export class VoiceAgent {
463
470
  })();
464
471
  }
465
472
 
473
+ // Native capture path (see `audioIO`): same optimistic 'listening' flip + gen guard as the browser
474
+ // path, minus the audio graph and Silero. Warm resume reuses the live capture handle.
475
+ async _startNative() {
476
+ if (this._capture) { // warm resume — capture kept alive across stop()
477
+ this._capture.setMuted?.(this._muted);
478
+ this.stt.open?.(); this.tts.warm?.().catch(() => {});
479
+ this._set('listening');
480
+ return;
481
+ }
482
+ const gen = ++this._pipelineGen;
483
+ this._set('listening');
484
+ let cap;
485
+ try { cap = await this.audioIO.startCapture((f32) => { if (gen === this._pipelineGen && !this._closed) this._onCapture(f32); }); }
486
+ catch (e) { if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: e?.message || 'Microphone access failed' }); } return; }
487
+ if (gen !== this._pipelineGen || this._closed) { cap.stop(); return; }
488
+ this._capture = cap; cap.setMuted?.(this._muted);
489
+ this._useBootstrap = true; this._sileroPending = false; // energy-VAD for the whole session
490
+ this.stt.open?.();
491
+ setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
492
+ }
493
+
466
494
  // True once the pipeline has been torn down (stop() / a fatal STT error). The host reconciles its
467
495
  // own "running" flag against this after an error so a self-stopped agent isn't treated as live.
468
496
  get closed() { return this._closed; }
@@ -490,6 +518,7 @@ export class VoiceAgent {
490
518
  // session) and before a cold rebuild (mic device switch). After this, start() pays the full cold
491
519
  // cost again. Split out so stop() can pause cheaply without duplicating this.
492
520
  _teardownPipeline() {
521
+ this._capture?.stop(); this._capture = null;
493
522
  this._vad?.destroy?.(); this._vad = null;
494
523
  this._node?.disconnect(); this._node = null;
495
524
  this._ctx?.close(); this._ctx = null;
@@ -578,7 +607,7 @@ export class VoiceAgent {
578
607
  // VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
579
608
  // or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
580
609
  setMuted(muted) {
581
- this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted);
610
+ this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted); this._capture?.setMuted?.(muted);
582
611
  if (muted) this._dropPrefetch(); // the utterance that seeded it just got cut off — no final will claim it
583
612
  // A self-capturing STT (Web Speech) has no agent stream to gate — it listens on its OWN mic, so
584
613
  // stop it explicitly via setEnabled or a "muted" agent keeps transcribing.
@@ -1476,6 +1505,11 @@ function sentenceEnd(s) {
1476
1505
  // context from the host's mic-button click (unlockAudio) and reuse it for all later playback.
1477
1506
  let sharedCtx = null;
1478
1507
  const audioCtx = () => (sharedCtx ??= new (window.AudioContext || window.webkitAudioContext)());
1508
+ // Host-supplied TTS playout (React Native etc.): `load(blob) → { duration, start(offsetSec) → clip }`
1509
+ // where clip has the WebAudioClip surface below (currentTime/duration/paused/ended/onended/onpause/pause).
1510
+ // Set once per process via setPlaybackIO(); null → shared Web Audio context + AEC loopback sink.
1511
+ let playbackIO = null;
1512
+ export function setPlaybackIO(io) { playbackIO = io; }
1479
1513
 
1480
1514
  // ── AEC loopback sink ────────────────────────────────────────────────────────
1481
1515
  // THE self-interruption root cause: the browser's echo canceller (getUserMedia
@@ -1630,6 +1664,7 @@ export class StreamingTTS {
1630
1664
  // forward/backward tap inside it), as a non-whitespace offset into `text`. abort always dominates seek.
1631
1665
  async _playBuf(wav, text, signal, prefix = '', startNw = 0) {
1632
1666
  if (signal?.aborted || !wav) return { reason: 'abort', heard: prefix };
1667
+ if (playbackIO) return this._playNative(wav, text, signal, prefix, startNw);
1633
1668
  const ctx = audioCtx();
1634
1669
  // Decode first (works while suspended) so we don't block synthesis on resume.
1635
1670
  let buffer;
@@ -1658,7 +1693,21 @@ export class StreamingTTS {
1658
1693
  if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
1659
1694
  const dur = buffer.duration;
1660
1695
  const offsetSec = startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0; // tap-into-clip → start offset
1661
- const clip = new WebAudioClip(ctx, buffer, offsetSec);
1696
+ return this._awaitClip(new WebAudioClip(ctx, buffer, offsetSec), text, signal, prefix, startNw, dur);
1697
+ }
1698
+
1699
+ async _playNative(wav, text, signal, prefix, startNw) {
1700
+ let src;
1701
+ try { src = await playbackIO.load(wav); }
1702
+ catch (e) { throw new Error(`Audio decode: ${e?.message || e}`); }
1703
+ if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
1704
+ if (this._seekPending) return { reason: 'seek', heard: prefix + sliceNw(text, startNw) };
1705
+ const dur = src.duration;
1706
+ return this._awaitClip(src.start(startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0), text, signal, prefix, startNw, dur);
1707
+ }
1708
+
1709
+ // Drive one started clip to a settled {reason, heard}: abort/seek/ended + the spoken-cursor ticks.
1710
+ _awaitClip(clip, text, signal, prefix, startNw, dur) {
1662
1711
  this._audio = clip;
1663
1712
  return new Promise((resolve) => {
1664
1713
  // Proportional position in NON-WHITESPACE space (the same domain seek() uses) → slice `text` to that