@todoforai/voiceloop 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +22 -0
- package/src/index.js +1 -1
- package/src/voice-agent.js +52 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.15",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -101,7 +101,29 @@ export interface VoiceTTS {
|
|
|
101
101
|
setOnProgress?(fn: TtsProgress | null): void;
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
/** Host-supplied mic capture (React Native, Electron, tests) — replaces getUserMedia + AudioWorklet +
|
|
105
|
+
* Silero. Frames: 16 kHz mono Float32, ~256 ms (4096 samples). The bootstrap energy-VAD runs for
|
|
106
|
+
* the whole session; pair with a continuous STT (Deepgram Flux) for server-side end-of-turn. */
|
|
107
|
+
export interface AudioIO {
|
|
108
|
+
startCapture(onFrame: (f32: Float32Array) => void): Promise<{ stop(): void; setMuted?(muted: boolean): void }>;
|
|
109
|
+
}
|
|
110
|
+
/** Minimal clip surface StreamingTTS drives: position for the spoken cursor, pause() for barge-in,
|
|
111
|
+
* onended (natural end) / onpause (stopped) callbacks. */
|
|
112
|
+
export interface PlaybackClip {
|
|
113
|
+
readonly currentTime: number; readonly duration: number; paused: boolean; ended: boolean;
|
|
114
|
+
onended: (() => void) | null; onpause: (() => void) | null;
|
|
115
|
+
pause(): void;
|
|
116
|
+
}
|
|
117
|
+
/** Host-supplied TTS playout — replaces the shared Web Audio context + AEC loopback sink. */
|
|
118
|
+
export interface PlaybackIO {
|
|
119
|
+
load(blob: Blob): Promise<{ duration: number; start(offsetSec: number): PlaybackClip }>;
|
|
120
|
+
}
|
|
121
|
+
/** Route ALL StreamingTTS playback through `io` (process-wide); null restores Web Audio. */
|
|
122
|
+
export function setPlaybackIO(io: PlaybackIO | null): void;
|
|
123
|
+
|
|
104
124
|
export interface VoiceAgentOptions {
|
|
125
|
+
/** Native mic capture; omit for the browser pipeline. See AudioIO. */
|
|
126
|
+
audioIO?: AudioIO;
|
|
105
127
|
sysmsg?: string;
|
|
106
128
|
/** Persona/system prompt prepended to the conversation context. Defaults to the built-in
|
|
107
129
|
* voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
|
package/src/index.js
CHANGED
package/src/voice-agent.js
CHANGED
|
@@ -172,7 +172,13 @@ export class VoiceAgent {
|
|
|
172
172
|
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
173
173
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
174
174
|
tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
|
|
175
|
-
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, onEvent = () => {} } = {}) {
|
|
175
|
+
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, audioIO = null, onEvent = () => {} } = {}) {
|
|
176
|
+
// Host-supplied mic capture (React Native, Electron main, tests): `startCapture(onFrame)` delivers
|
|
177
|
+
// 16 kHz mono Float32 frames (~256 ms) and resolves a handle with stop()/setMuted(). Replaces the
|
|
178
|
+
// whole getUserMedia + AudioWorklet + Silero path — the bootstrap energy-VAD then runs for the
|
|
179
|
+
// session (no ONNX/WASM on native), which is fine with a continuous STT (Deepgram Flux) that
|
|
180
|
+
// does end-of-turn itself; VAD only drives state events + barge-in reset there.
|
|
181
|
+
this.audioIO = audioIO; this._capture = null;
|
|
176
182
|
// Each agent gets its own PiperTTS (it owns an AudioContext + playing node) — a shared
|
|
177
183
|
// module-level default would make two agents fight over one audio output. `speed` scales
|
|
178
184
|
// playback rate (1.2 → 20% faster) on the default TTS; ignored if a custom `tts` is passed.
|
|
@@ -356,6 +362,7 @@ export class VoiceAgent {
|
|
|
356
362
|
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
357
363
|
return;
|
|
358
364
|
}
|
|
365
|
+
if (this.audioIO) return this._startNative();
|
|
359
366
|
// WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
|
|
360
367
|
// mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
|
|
361
368
|
// permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
|
|
@@ -463,6 +470,27 @@ export class VoiceAgent {
|
|
|
463
470
|
})();
|
|
464
471
|
}
|
|
465
472
|
|
|
473
|
+
// Native capture path (see `audioIO`): same optimistic 'listening' flip + gen guard as the browser
|
|
474
|
+
// path, minus the audio graph and Silero. Warm resume reuses the live capture handle.
|
|
475
|
+
async _startNative() {
|
|
476
|
+
if (this._capture) { // warm resume — capture kept alive across stop()
|
|
477
|
+
this._capture.setMuted?.(this._muted);
|
|
478
|
+
this.stt.open?.(); this.tts.warm?.().catch(() => {});
|
|
479
|
+
this._set('listening');
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const gen = ++this._pipelineGen;
|
|
483
|
+
this._set('listening');
|
|
484
|
+
let cap;
|
|
485
|
+
try { cap = await this.audioIO.startCapture((f32) => { if (gen === this._pipelineGen && !this._closed) this._onCapture(f32); }); }
|
|
486
|
+
catch (e) { if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: e?.message || 'Microphone access failed' }); } return; }
|
|
487
|
+
if (gen !== this._pipelineGen || this._closed) { cap.stop(); return; }
|
|
488
|
+
this._capture = cap; cap.setMuted?.(this._muted);
|
|
489
|
+
this._useBootstrap = true; this._sileroPending = false; // energy-VAD for the whole session
|
|
490
|
+
this.stt.open?.();
|
|
491
|
+
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
492
|
+
}
|
|
493
|
+
|
|
466
494
|
// True once the pipeline has been torn down (stop() / a fatal STT error). The host reconciles its
|
|
467
495
|
// own "running" flag against this after an error so a self-stopped agent isn't treated as live.
|
|
468
496
|
get closed() { return this._closed; }
|
|
@@ -490,6 +518,7 @@ export class VoiceAgent {
|
|
|
490
518
|
// session) and before a cold rebuild (mic device switch). After this, start() pays the full cold
|
|
491
519
|
// cost again. Split out so stop() can pause cheaply without duplicating this.
|
|
492
520
|
_teardownPipeline() {
|
|
521
|
+
this._capture?.stop(); this._capture = null;
|
|
493
522
|
this._vad?.destroy?.(); this._vad = null;
|
|
494
523
|
this._node?.disconnect(); this._node = null;
|
|
495
524
|
this._ctx?.close(); this._ctx = null;
|
|
@@ -578,7 +607,7 @@ export class VoiceAgent {
|
|
|
578
607
|
// VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
|
|
579
608
|
// or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
|
|
580
609
|
setMuted(muted) {
|
|
581
|
-
this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted);
|
|
610
|
+
this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted); this._capture?.setMuted?.(muted);
|
|
582
611
|
if (muted) this._dropPrefetch(); // the utterance that seeded it just got cut off — no final will claim it
|
|
583
612
|
// A self-capturing STT (Web Speech) has no agent stream to gate — it listens on its OWN mic, so
|
|
584
613
|
// stop it explicitly via setEnabled or a "muted" agent keeps transcribing.
|
|
@@ -1476,6 +1505,11 @@ function sentenceEnd(s) {
|
|
|
1476
1505
|
// context from the host's mic-button click (unlockAudio) and reuse it for all later playback.
|
|
1477
1506
|
let sharedCtx = null;
|
|
1478
1507
|
const audioCtx = () => (sharedCtx ??= new (window.AudioContext || window.webkitAudioContext)());
|
|
1508
|
+
// Host-supplied TTS playout (React Native etc.): `load(blob) → { duration, start(offsetSec) → clip }`
|
|
1509
|
+
// where clip has the WebAudioClip surface below (currentTime/duration/paused/ended/onended/onpause/pause).
|
|
1510
|
+
// Set once per process via setPlaybackIO(); null → shared Web Audio context + AEC loopback sink.
|
|
1511
|
+
let playbackIO = null;
|
|
1512
|
+
export function setPlaybackIO(io) { playbackIO = io; }
|
|
1479
1513
|
|
|
1480
1514
|
// ── AEC loopback sink ────────────────────────────────────────────────────────
|
|
1481
1515
|
// THE self-interruption root cause: the browser's echo canceller (getUserMedia
|
|
@@ -1630,6 +1664,7 @@ export class StreamingTTS {
|
|
|
1630
1664
|
// forward/backward tap inside it), as a non-whitespace offset into `text`. abort always dominates seek.
|
|
1631
1665
|
async _playBuf(wav, text, signal, prefix = '', startNw = 0) {
|
|
1632
1666
|
if (signal?.aborted || !wav) return { reason: 'abort', heard: prefix };
|
|
1667
|
+
if (playbackIO) return this._playNative(wav, text, signal, prefix, startNw);
|
|
1633
1668
|
const ctx = audioCtx();
|
|
1634
1669
|
// Decode first (works while suspended) so we don't block synthesis on resume.
|
|
1635
1670
|
let buffer;
|
|
@@ -1658,7 +1693,21 @@ export class StreamingTTS {
|
|
|
1658
1693
|
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1659
1694
|
const dur = buffer.duration;
|
|
1660
1695
|
const offsetSec = startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0; // tap-into-clip → start offset
|
|
1661
|
-
|
|
1696
|
+
return this._awaitClip(new WebAudioClip(ctx, buffer, offsetSec), text, signal, prefix, startNw, dur);
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
async _playNative(wav, text, signal, prefix, startNw) {
|
|
1700
|
+
let src;
|
|
1701
|
+
try { src = await playbackIO.load(wav); }
|
|
1702
|
+
catch (e) { throw new Error(`Audio decode: ${e?.message || e}`); }
|
|
1703
|
+
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1704
|
+
if (this._seekPending) return { reason: 'seek', heard: prefix + sliceNw(text, startNw) };
|
|
1705
|
+
const dur = src.duration;
|
|
1706
|
+
return this._awaitClip(src.start(startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0), text, signal, prefix, startNw, dur);
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
// Drive one started clip to a settled {reason, heard}: abort/seek/ended + the spoken-cursor ticks.
|
|
1710
|
+
_awaitClip(clip, text, signal, prefix, startNw, dur) {
|
|
1662
1711
|
this._audio = clip;
|
|
1663
1712
|
return new Promise((resolve) => {
|
|
1664
1713
|
// Proportional position in NON-WHITESPACE space (the same domain seek() uses) → slice `text` to that
|