@handsforbots/voice 0.0.0-stage → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +9 -0
- package/README.md +20 -2
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/player.d.ts +33 -0
- package/dist/player.d.ts.map +1 -0
- package/dist/player.js +141 -0
- package/dist/player.js.map +1 -0
- package/dist/providers/echo.d.ts +25 -0
- package/dist/providers/echo.d.ts.map +1 -0
- package/dist/providers/echo.js +85 -0
- package/dist/providers/echo.js.map +1 -0
- package/dist/providers/end-of-turn.d.ts +22 -0
- package/dist/providers/end-of-turn.d.ts.map +1 -0
- package/dist/providers/end-of-turn.js +29 -0
- package/dist/providers/end-of-turn.js.map +1 -0
- package/dist/providers/http.d.ts +48 -0
- package/dist/providers/http.d.ts.map +1 -0
- package/dist/providers/http.js +153 -0
- package/dist/providers/http.js.map +1 -0
- package/dist/providers/media.d.ts +15 -0
- package/dist/providers/media.d.ts.map +1 -0
- package/dist/providers/media.js +122 -0
- package/dist/providers/media.js.map +1 -0
- package/dist/providers/vosk-browser.d.ts +33 -0
- package/dist/providers/vosk-browser.d.ts.map +1 -0
- package/dist/providers/vosk-browser.js +82 -0
- package/dist/providers/vosk-browser.js.map +1 -0
- package/dist/providers/websocket.d.ts +38 -0
- package/dist/providers/websocket.d.ts.map +1 -0
- package/dist/providers/websocket.js +94 -0
- package/dist/providers/websocket.js.map +1 -0
- package/dist/providers/webspeech.d.ts +40 -0
- package/dist/providers/webspeech.d.ts.map +1 -0
- package/dist/providers/webspeech.js +201 -0
- package/dist/providers/webspeech.js.map +1 -0
- package/dist/types.d.ts +263 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +20 -0
- package/dist/types.js.map +1 -0
- package/dist/voice.d.ts +283 -0
- package/dist/voice.d.ts.map +1 -0
- package/dist/voice.js +1209 -0
- package/dist/voice.js.map +1 -0
- package/package.json +37 -3
package/dist/voice.js
ADDED
|
@@ -0,0 +1,1209 @@
|
|
|
1
|
+
import { definePlugin, textOf } from '@handsforbots/core';
|
|
2
|
+
import { planProsody, webAudioPlayer } from './player.js';
|
|
3
|
+
import { textEchoFilter } from './providers/echo.js';
|
|
4
|
+
import { SpeechError, } from './types.js';
|
|
5
|
+
const toArray = (value) => (value === undefined ? [] : Array.isArray(value) ? value : [value]);
|
|
6
|
+
const BACKCHANNELS = {
|
|
7
|
+
pt: ['aham', 'uhum', 'hum', 'hm', 'hmm', 'ah', 'ahn', 'é', 'tá', 'ta', 'ok', 'okay', 'sei', 'certo', 'entendi', 'beleza'],
|
|
8
|
+
en: ['uh huh', 'uh-huh', 'mhm', 'mm', 'hmm', 'hm', 'uh', 'um', 'ah', 'yeah', 'yep', 'ok', 'okay', 'right', 'sure', 'i see'],
|
|
9
|
+
es: ['ajá', 'aja', 'mhm', 'eh', 'ah', 'vale', 'ok', 'claro', 'ya'],
|
|
10
|
+
};
|
|
11
|
+
const normalize = (text) => text
|
|
12
|
+
.toLowerCase()
|
|
13
|
+
.replace(/[.,!?¡¿…;:"]/g, ' ')
|
|
14
|
+
.replace(/\s+/g, ' ')
|
|
15
|
+
.trim();
|
|
16
|
+
const wordsOf = (text) => normalize(text).split(' ').filter(Boolean);
|
|
17
|
+
/** End of the first complete sentence in `text` after `from`, or -1. */
|
|
18
|
+
function sentenceEnd(text, from) {
|
|
19
|
+
const pattern = /[.!?…]+["')\]]*(?=\s)|\n/g;
|
|
20
|
+
pattern.lastIndex = from;
|
|
21
|
+
const match = pattern.exec(text);
|
|
22
|
+
return match ? match.index + match[0].length : -1;
|
|
23
|
+
}
|
|
24
|
+
export const voice = definePlugin({
|
|
25
|
+
name: 'voice',
|
|
26
|
+
provides: ['voice'],
|
|
27
|
+
async apply(ctx, options = {}) {
|
|
28
|
+
const service = await createVoice(ctx, options);
|
|
29
|
+
ctx.provide('voice', service);
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
async function supportedOf(providers) {
|
|
33
|
+
const checks = await Promise.all(providers.map(async (p) => ({ p, ok: await Promise.resolve(p.isSupported()).catch(() => false) })));
|
|
34
|
+
return checks.filter((c) => c.ok).map((c) => c.p);
|
|
35
|
+
}
|
|
36
|
+
export async function createVoice(ctx, options) {
|
|
37
|
+
const language = options.language || (typeof document !== 'undefined' && document.documentElement.lang) || 'en-US';
|
|
38
|
+
const sttProviders = await supportedOf(toArray(options.stt));
|
|
39
|
+
const player = options.player ?? webAudioPlayer();
|
|
40
|
+
const playable = (p) => !!p.speak || (!!p.synthesize && player.isSupported());
|
|
41
|
+
let ttsProviders = (await supportedOf(toArray(options.tts))).filter(playable);
|
|
42
|
+
const allowed = (provider) => !provider.consent || ctx.app.consent.granted(provider.consent);
|
|
43
|
+
let bargeIn = options.bargeIn !== false;
|
|
44
|
+
let bargeInOptions = typeof options.bargeIn === 'object' ? options.bargeIn : {};
|
|
45
|
+
let timing = { ...options.turn };
|
|
46
|
+
const ms = {
|
|
47
|
+
silence: () => timing.silence ?? 800,
|
|
48
|
+
minSilence: () => timing.minSilence ?? 300,
|
|
49
|
+
maxSilence: () => timing.maxSilence ?? 2000,
|
|
50
|
+
threshold: () => timing.threshold ?? 0.5,
|
|
51
|
+
timeout: () => timing.timeout ?? 600,
|
|
52
|
+
};
|
|
53
|
+
const backchannels = () => new Set((bargeInOptions.ignore ?? BACKCHANNELS[language.slice(0, 2).toLowerCase()] ?? BACKCHANNELS.en).map(normalize));
|
|
54
|
+
const phaseOf = (s) => s.listening && s.partial && s.endOfTurnAt === undefined
|
|
55
|
+
? 'userSpeaking'
|
|
56
|
+
: s.endOfTurnAt !== undefined
|
|
57
|
+
? 'waitingEndOfTurn'
|
|
58
|
+
: s.speaking
|
|
59
|
+
? 'botSpeaking'
|
|
60
|
+
: s.listening
|
|
61
|
+
? 'listening'
|
|
62
|
+
: 'idle';
|
|
63
|
+
let state = {
|
|
64
|
+
supported: { stt: sttProviders.length > 0, tts: ttsProviders.length > 0 },
|
|
65
|
+
mode: options.mode ?? 'push-to-talk',
|
|
66
|
+
phase: 'idle',
|
|
67
|
+
listening: false,
|
|
68
|
+
speaking: false,
|
|
69
|
+
partial: '',
|
|
70
|
+
interrupted: false,
|
|
71
|
+
lastInput: 'keyboard',
|
|
72
|
+
output: options.output ?? 'auto',
|
|
73
|
+
stt: sttProviders[0]?.name,
|
|
74
|
+
tts: ttsProviders[0]?.name,
|
|
75
|
+
ttsStatus: {},
|
|
76
|
+
synthesizing: false,
|
|
77
|
+
effects: options.effects?.name,
|
|
78
|
+
};
|
|
79
|
+
let echo = options.echo === false ? undefined : (options.echo ?? textEchoFilter());
|
|
80
|
+
state.echo = echo?.name;
|
|
81
|
+
const listeners = new Set();
|
|
82
|
+
const set = (patch) => {
|
|
83
|
+
const next = { ...state, ...patch };
|
|
84
|
+
state = { ...next, phase: phaseOf(next) };
|
|
85
|
+
for (const listener of listeners)
|
|
86
|
+
listener();
|
|
87
|
+
};
|
|
88
|
+
const decide = (decision) => ctx.emit('voice.decision', decision);
|
|
89
|
+
/** Shows the error in the state (for the UI) and on the kernel bus (for the developer). */
|
|
90
|
+
const fail = (error, patch = {}, source = 'voice') => {
|
|
91
|
+
set({ ...patch, error });
|
|
92
|
+
ctx.emit('error', { error, source });
|
|
93
|
+
};
|
|
94
|
+
/* ---------------------------------------------------------------------- */
|
|
95
|
+
/* End-of-turn model */
|
|
96
|
+
/* ---------------------------------------------------------------------- */
|
|
97
|
+
let endOfTurn;
|
|
98
|
+
/** Ready to be asked: supported and loaded. */
|
|
99
|
+
let endOfTurnReady = false;
|
|
100
|
+
const setEndOfTurn = async (provider) => {
|
|
101
|
+
endOfTurn = provider ?? undefined;
|
|
102
|
+
endOfTurnReady = false;
|
|
103
|
+
set({ endOfTurn: endOfTurn?.name });
|
|
104
|
+
if (!provider)
|
|
105
|
+
return;
|
|
106
|
+
try {
|
|
107
|
+
if (!(await provider.isSupported()))
|
|
108
|
+
throw new Error('not supported');
|
|
109
|
+
await provider.load?.();
|
|
110
|
+
if (endOfTurn === provider)
|
|
111
|
+
endOfTurnReady = true;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
if (endOfTurn === provider)
|
|
115
|
+
decide({ kind: 'end-of-turn-unavailable', provider: provider.name, reason: error?.message ?? 'failed' });
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
void setEndOfTurn(options.endOfTurn);
|
|
119
|
+
/** Recent microphone audio, for audio models. */
|
|
120
|
+
let audioFrames = [];
|
|
121
|
+
let audioRate = 16000;
|
|
122
|
+
const keepAudio = (samples, sampleRate) => {
|
|
123
|
+
if (endOfTurn?.input !== 'audio')
|
|
124
|
+
return;
|
|
125
|
+
audioRate = sampleRate;
|
|
126
|
+
audioFrames.push(samples);
|
|
127
|
+
const max = (timing.audioSeconds ?? 8) * sampleRate;
|
|
128
|
+
let total = audioFrames.reduce((sum, f) => sum + f.length, 0);
|
|
129
|
+
while (total - audioFrames[0].length >= max)
|
|
130
|
+
total -= audioFrames.shift().length;
|
|
131
|
+
};
|
|
132
|
+
const recentAudio = () => {
|
|
133
|
+
const samples = new Float32Array(audioFrames.reduce((sum, f) => sum + f.length, 0));
|
|
134
|
+
let offset = 0;
|
|
135
|
+
for (const frame of audioFrames) {
|
|
136
|
+
samples.set(frame, offset);
|
|
137
|
+
offset += frame.length;
|
|
138
|
+
}
|
|
139
|
+
return { samples, sampleRate: audioRate };
|
|
140
|
+
};
|
|
141
|
+
const historyFor = (turns) => ctx.app.messages
|
|
142
|
+
.filter((m) => m.role === 'user' || m.role === 'assistant')
|
|
143
|
+
.map((m) => ({ role: m.role, text: textOf(m) }))
|
|
144
|
+
.filter((m) => m.text)
|
|
145
|
+
.slice(-turns);
|
|
146
|
+
let turn = { texts: [], live: '', heardAt: 0, pauseSince: 0 };
|
|
147
|
+
/** State with no pause being timed. */
|
|
148
|
+
const noPause = { pauseSince: undefined, endOfTurnAt: undefined, endOfTurnCheckAt: undefined };
|
|
149
|
+
let turnTimer;
|
|
150
|
+
let askTimer;
|
|
151
|
+
let asking;
|
|
152
|
+
const cache = new Map();
|
|
153
|
+
/** The user already cut the reply during this turn. */
|
|
154
|
+
let bargedIn = false;
|
|
155
|
+
/** Live words sent before the recognizer finalized them: their late final is dropped. */
|
|
156
|
+
let sentLive = '';
|
|
157
|
+
const turnText = (live = turn.live) => [...turn.texts, live].filter(Boolean).join(' ');
|
|
158
|
+
const clearTurnTimers = () => {
|
|
159
|
+
clearTimeout(turnTimer);
|
|
160
|
+
clearTimeout(askTimer);
|
|
161
|
+
asking?.abort();
|
|
162
|
+
asking = undefined;
|
|
163
|
+
};
|
|
164
|
+
const sendTurn = (reason, probability) => {
|
|
165
|
+
clearTurnTimers();
|
|
166
|
+
const text = turnText('');
|
|
167
|
+
const done = turn;
|
|
168
|
+
turn = { texts: [], live: turn.live, heardAt: turn.live ? turn.heardAt : 0, pauseSince: 0 };
|
|
169
|
+
bargedIn = false;
|
|
170
|
+
set({ ...noPause, partial: turn.live });
|
|
171
|
+
if (!text)
|
|
172
|
+
return;
|
|
173
|
+
decide({ kind: 'end-of-turn', text, reason, probability });
|
|
174
|
+
ctx.signal({
|
|
175
|
+
modality: 'transcript',
|
|
176
|
+
parts: [{ type: 'text', text }],
|
|
177
|
+
source: 'voice',
|
|
178
|
+
meta: { confidence: done.confidence, stt: done.stt, language },
|
|
179
|
+
});
|
|
180
|
+
};
|
|
181
|
+
const discardPending = () => {
|
|
182
|
+
clearTurnTimers();
|
|
183
|
+
turn = { texts: [], live: '', heardAt: 0, pauseSince: 0 };
|
|
184
|
+
bargedIn = false;
|
|
185
|
+
set({ ...noPause, partial: '' });
|
|
186
|
+
};
|
|
187
|
+
/** Waits after a final result, asking the model if there is one. The pause counts from the last words heard. */
|
|
188
|
+
const scheduleEndOfTurn = (sttDecides) => {
|
|
189
|
+
clearTurnTimers();
|
|
190
|
+
const at = turn.pauseSince;
|
|
191
|
+
const model = !sttDecides && endOfTurn && endOfTurnReady && allowed(endOfTurn) && (endOfTurn.input === 'text' || audioFrames.length > 0)
|
|
192
|
+
? endOfTurn
|
|
193
|
+
: undefined;
|
|
194
|
+
/** Publishes the whole pause in one update, so the UI never sees half of it. */
|
|
195
|
+
const wait = (delay, reason, checkAt) => {
|
|
196
|
+
clearTimeout(turnTimer);
|
|
197
|
+
const deadline = at + delay;
|
|
198
|
+
set({ pauseSince: at, endOfTurnAt: deadline, endOfTurnCheckAt: checkAt });
|
|
199
|
+
turnTimer = setTimeout(() => sendTurn(reason), Math.max(0, deadline - Date.now()));
|
|
200
|
+
};
|
|
201
|
+
if (!model)
|
|
202
|
+
return wait(sttDecides ? ms.maxSilence() : ms.silence(), sttDecides ? 'max-silence' : 'silence');
|
|
203
|
+
wait(ms.maxSilence(), 'max-silence', at + ms.minSilence());
|
|
204
|
+
askTimer = setTimeout(async () => {
|
|
205
|
+
const text = turnText('');
|
|
206
|
+
const controller = new AbortController();
|
|
207
|
+
asking = controller;
|
|
208
|
+
let probability = cache.get(text);
|
|
209
|
+
if (probability === undefined) {
|
|
210
|
+
const timeout = new Promise((resolve) => setTimeout(resolve, ms.timeout()));
|
|
211
|
+
const prediction = Promise.resolve()
|
|
212
|
+
.then(() => model.predict({
|
|
213
|
+
partial: text,
|
|
214
|
+
history: historyFor(model.historyTurns ?? 6),
|
|
215
|
+
language,
|
|
216
|
+
audio: model.input === 'audio' ? recentAudio() : undefined,
|
|
217
|
+
}, controller.signal))
|
|
218
|
+
.catch((error) => {
|
|
219
|
+
if (!controller.signal.aborted)
|
|
220
|
+
ctx.emit('error', { error, source: `voice:end-of-turn:${model.name}` });
|
|
221
|
+
return undefined;
|
|
222
|
+
});
|
|
223
|
+
probability = await Promise.race([prediction, timeout]);
|
|
224
|
+
if (probability !== undefined) {
|
|
225
|
+
cache.clear();
|
|
226
|
+
cache.set(text, probability);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (controller.signal.aborted || asking !== controller)
|
|
230
|
+
return;
|
|
231
|
+
asking = undefined;
|
|
232
|
+
if (probability === undefined) {
|
|
233
|
+
controller.abort();
|
|
234
|
+
wait(ms.silence(), 'silence'); // the model failed or is slow: silence decides (often at once)
|
|
235
|
+
}
|
|
236
|
+
else if (probability >= ms.threshold()) {
|
|
237
|
+
sendTurn('model', probability);
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
set({ endOfTurnCheckAt: undefined }); // not finished: endOfTurnAt is now exact
|
|
241
|
+
}
|
|
242
|
+
}, Math.max(0, at + ms.minSilence() - Date.now()));
|
|
243
|
+
};
|
|
244
|
+
/* ---------------------------------------------------------------------- */
|
|
245
|
+
/* Echo: the bot's own speech coming back through the microphone */
|
|
246
|
+
/* ---------------------------------------------------------------------- */
|
|
247
|
+
/** Sentences spoken recently, oldest first. */
|
|
248
|
+
let speech = [];
|
|
249
|
+
const recentSpeech = () => {
|
|
250
|
+
const since = Date.now() - (echo?.tail ?? 2000);
|
|
251
|
+
speech = speech.filter((s) => s.endedAt === undefined || s.endedAt >= since);
|
|
252
|
+
return speech;
|
|
253
|
+
};
|
|
254
|
+
/** What the user said in `text`, without the bot's voice ('' when it was all echo). */
|
|
255
|
+
const withoutEcho = (text, final) => {
|
|
256
|
+
if (!echo || !text.trim())
|
|
257
|
+
return text;
|
|
258
|
+
const recent = recentSpeech();
|
|
259
|
+
if (recent.length === 0)
|
|
260
|
+
return text;
|
|
261
|
+
try {
|
|
262
|
+
return echo.filter({ text, final, speech: [...recent], language });
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
ctx.emit('error', { error, source: `voice:echo:${echo.name}` });
|
|
266
|
+
return text;
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
/* ---------------------------------------------------------------------- */
|
|
270
|
+
/* Barge-in */
|
|
271
|
+
/* ---------------------------------------------------------------------- */
|
|
272
|
+
/** The bot is answering: speaking, or showing a reply as it streams. */
|
|
273
|
+
const botAnswering = () => {
|
|
274
|
+
if (state.speaking)
|
|
275
|
+
return true;
|
|
276
|
+
if (!ctx.app.busy)
|
|
277
|
+
return false;
|
|
278
|
+
const last = ctx.app.messages.at(-1);
|
|
279
|
+
return last?.role === 'assistant' && !!last.streaming && !!textOf(last);
|
|
280
|
+
};
|
|
281
|
+
/** Reactions to the reply being spoken, sent with the next turn (ADR 0015). */
|
|
282
|
+
let reactions;
|
|
283
|
+
const react = (text) => {
|
|
284
|
+
const message = ctx.app.messages.at(-1);
|
|
285
|
+
const messageId = message?.role === 'assistant' ? message.id : undefined;
|
|
286
|
+
const at = messageId === undefined ? undefined : (tracks.get(messageId)?.spoken ?? 0);
|
|
287
|
+
decide({ kind: 'ignore', text, reason: 'backchannel', ...(messageId === undefined ? {} : { messageId, at }) });
|
|
288
|
+
if (options.reactions === false || messageId === undefined || at === undefined)
|
|
289
|
+
return;
|
|
290
|
+
if (reactions?.messageId !== messageId || !ctx.app.context.some((s) => s.key === 'voice.reactions')) {
|
|
291
|
+
reactions = { messageId, reactions: [] };
|
|
292
|
+
}
|
|
293
|
+
reactions.reactions.push({ at, text });
|
|
294
|
+
ctx.app.signal({
|
|
295
|
+
kind: 'context',
|
|
296
|
+
key: 'voice.reactions',
|
|
297
|
+
once: true,
|
|
298
|
+
modality: 'transcript',
|
|
299
|
+
source: 'voice',
|
|
300
|
+
parts: [{ type: 'data', name: 'reactions', value: structuredClone(reactions) }],
|
|
301
|
+
});
|
|
302
|
+
};
|
|
303
|
+
/**
|
|
304
|
+
* Whether speech heard while the bot answers should cut it. Returns false
|
|
305
|
+
* for text to drop (a backchannel over the bot).
|
|
306
|
+
*/
|
|
307
|
+
const bargeInCheck = (text, final) => {
|
|
308
|
+
if (!bargeIn || bargedIn || !botAnswering())
|
|
309
|
+
return 'none';
|
|
310
|
+
const words = wordsOf(text);
|
|
311
|
+
if (words.length === 0)
|
|
312
|
+
return 'pending';
|
|
313
|
+
const ignore = backchannels();
|
|
314
|
+
if (ignore.has(words.join(' ')) || words.every((w) => ignore.has(w))) {
|
|
315
|
+
if (final)
|
|
316
|
+
react(text);
|
|
317
|
+
return final ? 'ignored' : 'pending';
|
|
318
|
+
}
|
|
319
|
+
if (!final && words.length < (bargeInOptions.minWords ?? 2))
|
|
320
|
+
return 'pending';
|
|
321
|
+
bargedIn = true;
|
|
322
|
+
decide({ kind: 'interrupt', text });
|
|
323
|
+
// Interrupt first: the interceptor reads how much was spoken before speech stops.
|
|
324
|
+
void ctx.app.interrupt({ by: 'voice' });
|
|
325
|
+
service.cancelSpeech();
|
|
326
|
+
set({ interrupted: true });
|
|
327
|
+
return 'interrupted';
|
|
328
|
+
};
|
|
329
|
+
/* ---------------------------------------------------------------------- */
|
|
330
|
+
/* Recognition sessions */
|
|
331
|
+
/* ---------------------------------------------------------------------- */
|
|
332
|
+
let sttIndex = 0;
|
|
333
|
+
let active;
|
|
334
|
+
/** The press in progress, while the user still holds. */
|
|
335
|
+
let holding;
|
|
336
|
+
let wantListening = false;
|
|
337
|
+
let pausedForSpeech = false;
|
|
338
|
+
let restartTimer;
|
|
339
|
+
const heldText = (held, partial) => [...held.texts, partial].filter(Boolean).join(' ');
|
|
340
|
+
/** Ends a hold-to-talk press: sends what was said, once. */
|
|
341
|
+
const release = (held, send = true) => {
|
|
342
|
+
if (holding === held)
|
|
343
|
+
holding = undefined;
|
|
344
|
+
if (held.done)
|
|
345
|
+
return;
|
|
346
|
+
held.done = true;
|
|
347
|
+
const text = heldText(held);
|
|
348
|
+
if (!send || !text)
|
|
349
|
+
return;
|
|
350
|
+
ctx.signal({
|
|
351
|
+
modality: 'transcript',
|
|
352
|
+
parts: [{ type: 'text', text }],
|
|
353
|
+
source: 'voice',
|
|
354
|
+
meta: { confidence: held.confidence, stt: held.stt, language },
|
|
355
|
+
});
|
|
356
|
+
};
|
|
357
|
+
const nextProvider = () => {
|
|
358
|
+
while (sttIndex < sttProviders.length && !allowed(sttProviders[sttIndex]))
|
|
359
|
+
sttIndex++;
|
|
360
|
+
return sttProviders[sttIndex];
|
|
361
|
+
};
|
|
362
|
+
const hasFallbackAfter = () => sttProviders.slice(sttIndex + 1).some(allowed);
|
|
363
|
+
const startSession = async () => {
|
|
364
|
+
const provider = nextProvider();
|
|
365
|
+
if (!provider) {
|
|
366
|
+
fail(new SpeechError('not-supported', 'No speech recognition available'), { listening: false });
|
|
367
|
+
wantListening = false;
|
|
368
|
+
if (holding)
|
|
369
|
+
release(holding);
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const handsFree = state.mode === 'hands-free';
|
|
373
|
+
const entry = { stopping: false, held: holding };
|
|
374
|
+
active = entry;
|
|
375
|
+
const isCurrent = () => active === entry;
|
|
376
|
+
set({
|
|
377
|
+
listening: true,
|
|
378
|
+
error: undefined,
|
|
379
|
+
stt: provider.name,
|
|
380
|
+
partial: entry.held ? heldText(entry.held) : handsFree ? turnText() : '',
|
|
381
|
+
});
|
|
382
|
+
let failed;
|
|
383
|
+
try {
|
|
384
|
+
entry.session = await provider.listen({ language, continuous: handsFree || !!entry.held }, {
|
|
385
|
+
onPartial(heard) {
|
|
386
|
+
if (!isCurrent() || entry.stopping)
|
|
387
|
+
return;
|
|
388
|
+
const text = withoutEcho(heard, false);
|
|
389
|
+
if (entry.held)
|
|
390
|
+
return set({ partial: heldText(entry.held, text) });
|
|
391
|
+
bargeInCheck(text, false);
|
|
392
|
+
if (!handsFree)
|
|
393
|
+
return set({ partial: text });
|
|
394
|
+
if (text.trim() && (text !== turn.live || !turn.heardAt))
|
|
395
|
+
turn.heardAt = Date.now();
|
|
396
|
+
turn.live = text;
|
|
397
|
+
if (!text.trim())
|
|
398
|
+
return set({ partial: turnText() });
|
|
399
|
+
// The user keeps talking: the pending turn waits for the next final result,
|
|
400
|
+
// or goes out with these words if the recognizer never delivers one.
|
|
401
|
+
if (turn.texts.length > 0) {
|
|
402
|
+
clearTurnTimers();
|
|
403
|
+
turnTimer = setTimeout(() => service.endTurn(), Math.max(0, turn.heardAt + ms.maxSilence() - Date.now()));
|
|
404
|
+
}
|
|
405
|
+
set({ ...noPause, partial: turnText() });
|
|
406
|
+
},
|
|
407
|
+
onFinal(text, meta) {
|
|
408
|
+
if (entry.discarded)
|
|
409
|
+
return;
|
|
410
|
+
const heard = text.trim();
|
|
411
|
+
const trimmed = withoutEcho(heard, true).trim();
|
|
412
|
+
if (heard && !trimmed) {
|
|
413
|
+
decide({ kind: 'ignore', text: heard, reason: 'echo' });
|
|
414
|
+
if (handsFree && !entry.held) {
|
|
415
|
+
turn.live = '';
|
|
416
|
+
turn.heardAt = 0;
|
|
417
|
+
}
|
|
418
|
+
if (isCurrent())
|
|
419
|
+
set({ partial: entry.held ? heldText(entry.held) : handsFree ? turnText() : '' });
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const held = entry.held;
|
|
423
|
+
if (held) {
|
|
424
|
+
if (isCurrent())
|
|
425
|
+
set({ partial: '' });
|
|
426
|
+
if (held.done || !trimmed)
|
|
427
|
+
return;
|
|
428
|
+
if (bargeInCheck(trimmed, true) === 'ignored')
|
|
429
|
+
return;
|
|
430
|
+
held.texts.push(trimmed);
|
|
431
|
+
held.stt = provider.name;
|
|
432
|
+
if (meta?.confidence !== undefined)
|
|
433
|
+
held.confidence = Math.min(held.confidence ?? 1, meta.confidence);
|
|
434
|
+
if (isCurrent() && !entry.stopping)
|
|
435
|
+
set({ partial: heldText(held) });
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
if (!trimmed)
|
|
439
|
+
return;
|
|
440
|
+
if (sentLive) {
|
|
441
|
+
const duplicate = normalize(trimmed) === sentLive;
|
|
442
|
+
sentLive = '';
|
|
443
|
+
if (duplicate)
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
if (bargeInCheck(trimmed, true) === 'ignored') {
|
|
447
|
+
if (handsFree) {
|
|
448
|
+
turn.live = '';
|
|
449
|
+
turn.heardAt = 0;
|
|
450
|
+
}
|
|
451
|
+
if (isCurrent())
|
|
452
|
+
set({ partial: handsFree ? turnText() : '' });
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
if (!handsFree) {
|
|
456
|
+
// A stopped session may still deliver the last utterance: keep it.
|
|
457
|
+
if (isCurrent())
|
|
458
|
+
set({ partial: '' });
|
|
459
|
+
ctx.signal({
|
|
460
|
+
modality: 'transcript',
|
|
461
|
+
parts: [{ type: 'text', text: trimmed }],
|
|
462
|
+
source: 'voice',
|
|
463
|
+
meta: { confidence: meta?.confidence, stt: provider.name, language },
|
|
464
|
+
});
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
turn.texts.push(trimmed);
|
|
468
|
+
turn.live = '';
|
|
469
|
+
turn.stt = provider.name;
|
|
470
|
+
// Recognizers deliver the final result some time after the user stops
|
|
471
|
+
// talking: the pause starts at the last words heard, not now.
|
|
472
|
+
turn.pauseSince = turn.heardAt || Date.now();
|
|
473
|
+
turn.heardAt = 0;
|
|
474
|
+
if (meta?.confidence !== undefined)
|
|
475
|
+
turn.confidence = Math.min(turn.confidence ?? 1, meta.confidence);
|
|
476
|
+
set({ partial: turnText() });
|
|
477
|
+
const sttDecides = !!provider.capabilities.endOfTurn;
|
|
478
|
+
if (entry.stopping || !wantListening || (sttDecides && meta?.endOfTurn === true)) {
|
|
479
|
+
sendTurn(sttDecides && meta?.endOfTurn ? 'stt' : 'manual');
|
|
480
|
+
}
|
|
481
|
+
else {
|
|
482
|
+
scheduleEndOfTurn(sttDecides);
|
|
483
|
+
}
|
|
484
|
+
},
|
|
485
|
+
onAudio(samples, sampleRate) {
|
|
486
|
+
if (isCurrent() && !entry.stopping)
|
|
487
|
+
keepAudio(samples, sampleRate);
|
|
488
|
+
},
|
|
489
|
+
onError(error) {
|
|
490
|
+
failed = error;
|
|
491
|
+
if (isCurrent() && error.code !== 'no-speech')
|
|
492
|
+
fail(error);
|
|
493
|
+
},
|
|
494
|
+
onEnd() {
|
|
495
|
+
clearTimeout(entry.killTimer);
|
|
496
|
+
if (!isCurrent()) {
|
|
497
|
+
if (entry.held && entry.held !== holding)
|
|
498
|
+
release(entry.held);
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
active = undefined;
|
|
502
|
+
const hasFallback = !!failed?.recoverable && hasFallbackAfter();
|
|
503
|
+
// Asking again cannot help: the user has to change a permission or a setting first.
|
|
504
|
+
if (failed?.code === 'not-allowed' || (failed?.code === 'service-not-allowed' && !hasFallback)) {
|
|
505
|
+
wantListening = false;
|
|
506
|
+
}
|
|
507
|
+
// Hold-to-talk outlives the browser's session: start another until release.
|
|
508
|
+
const keepHolding = !!entry.held && entry.held === holding && wantListening && !pausedForSpeech;
|
|
509
|
+
turn.live = '';
|
|
510
|
+
set({
|
|
511
|
+
listening: keepHolding,
|
|
512
|
+
partial: keepHolding && entry.held ? heldText(entry.held) : handsFree ? turnText() : '',
|
|
513
|
+
});
|
|
514
|
+
if (hasFallback) {
|
|
515
|
+
sttIndex++;
|
|
516
|
+
if (wantListening)
|
|
517
|
+
void startSession();
|
|
518
|
+
else if (entry.held)
|
|
519
|
+
release(entry.held);
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
if (keepHolding) {
|
|
523
|
+
restartTimer = setTimeout(() => wantListening && !active && void startSession(), 250);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
if (entry.held)
|
|
527
|
+
release(entry.held);
|
|
528
|
+
// Browsers end recognition after silence; hands-free keeps going.
|
|
529
|
+
if (wantListening && state.mode === 'hands-free' && !pausedForSpeech) {
|
|
530
|
+
restartTimer = setTimeout(() => wantListening && !active && void startSession(), 250);
|
|
531
|
+
}
|
|
532
|
+
else if (state.mode === 'push-to-talk') {
|
|
533
|
+
wantListening = false;
|
|
534
|
+
}
|
|
535
|
+
else if (!wantListening && turn.texts.length > 0) {
|
|
536
|
+
sendTurn('manual');
|
|
537
|
+
}
|
|
538
|
+
},
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
catch (error) {
|
|
542
|
+
const speechError = error instanceof SpeechError ? error : new SpeechError('unknown', error?.message);
|
|
543
|
+
if (isCurrent())
|
|
544
|
+
active = undefined;
|
|
545
|
+
fail(speechError, { listening: false });
|
|
546
|
+
if (speechError.recoverable && hasFallbackAfter()) {
|
|
547
|
+
sttIndex++;
|
|
548
|
+
return startSession();
|
|
549
|
+
}
|
|
550
|
+
wantListening = false;
|
|
551
|
+
if (entry.held)
|
|
552
|
+
release(entry.held);
|
|
553
|
+
}
|
|
554
|
+
};
|
|
555
|
+
/* ---------------------------------------------------------------------- */
|
|
556
|
+
/* Speech output, sentence by sentence */
|
|
557
|
+
/* ---------------------------------------------------------------------- */
|
|
558
|
+
let disposed = false;
|
|
559
|
+
let effects = options.effects ?? undefined;
|
|
560
|
+
let unfollowEffects;
|
|
561
|
+
const followEffects = () => {
|
|
562
|
+
unfollowEffects?.();
|
|
563
|
+
const current = effects;
|
|
564
|
+
unfollowEffects = current?.subscribe?.(() => {
|
|
565
|
+
if (state.effects !== current.name)
|
|
566
|
+
set({ effects: current.name });
|
|
567
|
+
});
|
|
568
|
+
if (state.effects !== current?.name)
|
|
569
|
+
set({ effects: current?.name });
|
|
570
|
+
};
|
|
571
|
+
const loadStart = options.load?.start ?? 'first-use';
|
|
572
|
+
const whileLoading = options.load?.whileLoading ?? 'fallback';
|
|
573
|
+
/* Models: when they download and which provider speaks meanwhile. */
|
|
574
|
+
const statuses = new Map();
|
|
575
|
+
/* Debugging: each provider's loading and internals go to `voice.tts`. */
|
|
576
|
+
const trace = (p, event) => {
|
|
577
|
+
const defined = Object.fromEntries(Object.entries(event).filter(([, value]) => value !== undefined));
|
|
578
|
+
ctx.emit('voice.tts', { ...defined, provider: p.name });
|
|
579
|
+
};
|
|
580
|
+
const traced = new Map();
|
|
581
|
+
/** Follows the `trace` of the current providers (called again by `setTTS`). */
|
|
582
|
+
const followTraces = (providers) => {
|
|
583
|
+
for (const [p, off] of traced) {
|
|
584
|
+
if (providers.includes(p))
|
|
585
|
+
continue;
|
|
586
|
+
off();
|
|
587
|
+
traced.delete(p);
|
|
588
|
+
}
|
|
589
|
+
for (const p of providers) {
|
|
590
|
+
if (!p.trace || traced.has(p))
|
|
591
|
+
continue;
|
|
592
|
+
try {
|
|
593
|
+
traced.set(p, p.trace((event) => trace(p, event)));
|
|
594
|
+
}
|
|
595
|
+
catch (error) {
|
|
596
|
+
ctx.emit('error', { error, source: `voice:tts:${p.name}` });
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
};
|
|
600
|
+
followTraces(ttsProviders);
|
|
601
|
+
const loads = new Map();
|
|
602
|
+
const loadControllers = new Set();
|
|
603
|
+
/** The loading policy has been triggered: failed downloads move on to the next model. */
|
|
604
|
+
let loadingWanted = false;
|
|
605
|
+
const statusOf = (p) => statuses.get(p) ?? (p.load ? { state: 'unloaded', total: p.size } : { state: 'ready' });
|
|
606
|
+
const publishStatus = () => set({ ttsStatus: Object.fromEntries(ttsProviders.map((p) => [p.name, statusOf(p)])) });
|
|
607
|
+
const setStatus = (p, status) => {
|
|
608
|
+
statuses.set(p, status);
|
|
609
|
+
publishStatus();
|
|
610
|
+
};
|
|
611
|
+
const ready = (p) => statusOf(p).state === 'ready';
|
|
612
|
+
const usable = (p) => allowed(p) && statusOf(p).state !== 'failed';
|
|
613
|
+
/** The provider speech should come from, once it is ready. */
|
|
614
|
+
const preferred = () => ttsProviders.find(usable);
|
|
615
|
+
const loadProvider = (p) => {
|
|
616
|
+
if (!p.load || ready(p))
|
|
617
|
+
return Promise.resolve(true);
|
|
618
|
+
const running = loads.get(p);
|
|
619
|
+
if (running)
|
|
620
|
+
return running;
|
|
621
|
+
const controller = new AbortController();
|
|
622
|
+
loadControllers.add(controller);
|
|
623
|
+
let lastProgress = 0;
|
|
624
|
+
/** Quarters of the download already reported on `voice.tts`. */
|
|
625
|
+
let quarter = 0;
|
|
626
|
+
const startedAt = Date.now();
|
|
627
|
+
trace(p, { event: 'load.start', size: p.size });
|
|
628
|
+
setStatus(p, { state: 'loading', loaded: 0, total: p.size });
|
|
629
|
+
const promise = p
|
|
630
|
+
.load((progress) => {
|
|
631
|
+
if (controller.signal.aborted)
|
|
632
|
+
return;
|
|
633
|
+
const total = progress.total ?? p.size;
|
|
634
|
+
const reached = total ? Math.min(3, Math.floor((progress.loaded / total) * 4)) : 0;
|
|
635
|
+
if (reached > quarter) {
|
|
636
|
+
quarter = reached;
|
|
637
|
+
trace(p, { event: 'load.progress', loaded: progress.loaded, total, file: progress.file });
|
|
638
|
+
}
|
|
639
|
+
const now = Date.now();
|
|
640
|
+
if (now - lastProgress < 100)
|
|
641
|
+
return;
|
|
642
|
+
lastProgress = now;
|
|
643
|
+
setStatus(p, { state: 'loading', loaded: progress.loaded, total });
|
|
644
|
+
}, controller.signal)
|
|
645
|
+
.then(() => {
|
|
646
|
+
if (controller.signal.aborted) {
|
|
647
|
+
trace(p, { event: 'load.aborted', ms: Date.now() - startedAt });
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
trace(p, { event: 'load.done', ms: Date.now() - startedAt });
|
|
651
|
+
setStatus(p, { state: 'ready', loaded: statusOf(p).loaded, total: statusOf(p).total });
|
|
652
|
+
return true;
|
|
653
|
+
})
|
|
654
|
+
.catch((error) => {
|
|
655
|
+
if (controller.signal.aborted) {
|
|
656
|
+
trace(p, { event: 'load.aborted', ms: Date.now() - startedAt });
|
|
657
|
+
return false;
|
|
658
|
+
}
|
|
659
|
+
const message = error?.message ?? String(error);
|
|
660
|
+
trace(p, { event: 'load.error', ms: Date.now() - startedAt, message });
|
|
661
|
+
setStatus(p, { state: 'failed', error: message });
|
|
662
|
+
ctx.emit('error', { error, source: `voice:tts:${p.name}` });
|
|
663
|
+
if (loadingWanted)
|
|
664
|
+
void loadPreferred();
|
|
665
|
+
return false;
|
|
666
|
+
})
|
|
667
|
+
.finally(() => {
|
|
668
|
+
loads.delete(p);
|
|
669
|
+
loadControllers.delete(controller);
|
|
670
|
+
});
|
|
671
|
+
loads.set(p, promise);
|
|
672
|
+
return promise;
|
|
673
|
+
};
|
|
674
|
+
const loadPreferred = () => {
|
|
675
|
+
loadingWanted = true;
|
|
676
|
+
const p = preferred();
|
|
677
|
+
return p ? loadProvider(p) : Promise.resolve(false);
|
|
678
|
+
};
|
|
679
|
+
/**
|
|
680
|
+
* Who speaks the next sentence; `undefined` = nobody (silent until the model
|
|
681
|
+
* is ready). Synchronous unless it has to wait for a download.
|
|
682
|
+
*/
|
|
683
|
+
const choose = () => {
|
|
684
|
+
const first = preferred();
|
|
685
|
+
if (!first || ready(first))
|
|
686
|
+
return first;
|
|
687
|
+
if (loadStart === 'first-use')
|
|
688
|
+
void loadPreferred();
|
|
689
|
+
// What happens to this chunk while the model is not ready, for the log.
|
|
690
|
+
const meanwhile = (action, to) => trace(first, { event: 'speech.while-loading', policy: whileLoading, action, to, state: statusOf(first).state });
|
|
691
|
+
switch (whileLoading) {
|
|
692
|
+
case 'fallback': {
|
|
693
|
+
const other = ttsProviders.find((p) => p !== first && usable(p) && ready(p));
|
|
694
|
+
meanwhile(other ? 'fallback' : 'silent', other?.name);
|
|
695
|
+
return other;
|
|
696
|
+
}
|
|
697
|
+
case 'wait': {
|
|
698
|
+
// Only a download in progress is waited for: with `manual`, nothing speaks before `loadSpeech()`.
|
|
699
|
+
const running = loads.get(first);
|
|
700
|
+
meanwhile(running ? 'wait' : 'silent');
|
|
701
|
+
return running?.then(() => choose());
|
|
702
|
+
}
|
|
703
|
+
case 'silent':
|
|
704
|
+
meanwhile('silent');
|
|
705
|
+
return undefined;
|
|
706
|
+
default: {
|
|
707
|
+
const never = whileLoading;
|
|
708
|
+
throw new Error(`Unknown whileLoading: ${String(never)}`);
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
};
|
|
712
|
+
switch (loadStart) {
|
|
713
|
+
case 'eager':
|
|
714
|
+
void loadPreferred();
|
|
715
|
+
break;
|
|
716
|
+
case 'idle': {
|
|
717
|
+
const idle = globalThis.requestIdleCallback;
|
|
718
|
+
const start = () => !disposed && void loadPreferred();
|
|
719
|
+
if (idle)
|
|
720
|
+
idle(start, { timeout: 5000 });
|
|
721
|
+
else
|
|
722
|
+
setTimeout(start, 1500);
|
|
723
|
+
break;
|
|
724
|
+
}
|
|
725
|
+
case 'first-use':
|
|
726
|
+
// Speech is how this app answers: the model is needed from the start.
|
|
727
|
+
if (options.output === 'voice')
|
|
728
|
+
void loadPreferred();
|
|
729
|
+
break;
|
|
730
|
+
case 'manual':
|
|
731
|
+
break;
|
|
732
|
+
default: {
|
|
733
|
+
const never = loadStart;
|
|
734
|
+
throw new Error(`Unknown load.start: ${String(never)}`);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
/* Audio output: browsers allow it only after a user gesture. */
|
|
738
|
+
const unlockAudio = () => {
|
|
739
|
+
if (player.isSupported() && ttsProviders.some((p) => p.synthesize))
|
|
740
|
+
player.unlock();
|
|
741
|
+
for (const provider of ttsProviders) {
|
|
742
|
+
try {
|
|
743
|
+
provider.unlock?.();
|
|
744
|
+
}
|
|
745
|
+
catch (error) {
|
|
746
|
+
ctx.emit('error', { error, source: `voice:tts:${provider.name}` });
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
};
|
|
750
|
+
const gestures = ['pointerdown', 'keydown', 'touchend'];
|
|
751
|
+
if (typeof document !== 'undefined') {
|
|
752
|
+
for (const type of gestures)
|
|
753
|
+
document.addEventListener(type, unlockAudio, { capture: true, passive: true });
|
|
754
|
+
}
|
|
755
|
+
/* Sentences: synthesis runs ahead of playback, one at a time. */
|
|
756
|
+
let speechController;
|
|
757
|
+
let speechQueue = Promise.resolve();
|
|
758
|
+
let synthQueue = Promise.resolve();
|
|
759
|
+
let synthesizing = 0;
|
|
760
|
+
/** Chunks queued and not finished: `speaking` stays on between sentences. */
|
|
761
|
+
let chunksLeft = 0;
|
|
762
|
+
const tracks = new Map();
|
|
763
|
+
const viaPlayer = (p) => !!p.synthesize && player.isSupported();
|
|
764
|
+
const synthesize = (provider, text, speakOptions, signal) => {
|
|
765
|
+
const job = synthQueue.then(async () => {
|
|
766
|
+
if (signal.aborted)
|
|
767
|
+
return undefined;
|
|
768
|
+
set({ synthesizing: ++synthesizing > 0 });
|
|
769
|
+
try {
|
|
770
|
+
return await provider.synthesize(text, speakOptions);
|
|
771
|
+
}
|
|
772
|
+
finally {
|
|
773
|
+
set({ synthesizing: --synthesizing > 0 });
|
|
774
|
+
}
|
|
775
|
+
});
|
|
776
|
+
synthQueue = job.catch(() => undefined);
|
|
777
|
+
return job;
|
|
778
|
+
};
|
|
779
|
+
const plainSpeech = (provider, signal) => {
|
|
780
|
+
const plan = planProsody(provider.prosody, effects?.prosody?.(), false);
|
|
781
|
+
return { provider, plan, options: { language, voice: options.voiceName, signal, prosody: plan.synth } };
|
|
782
|
+
};
|
|
783
|
+
/**
|
|
784
|
+
* Picks the provider and, for models and cloud audio, starts synthesizing
|
|
785
|
+
* right away. Providers that speak by themselves are prepared synchronously,
|
|
786
|
+
* so their speech starts in the same tick as before there were models.
|
|
787
|
+
*/
|
|
788
|
+
const prepare = (text, signal) => {
|
|
789
|
+
const chosen = choose();
|
|
790
|
+
if (chosen instanceof Promise)
|
|
791
|
+
return chosen.then((p) => (p ? synthesizeWith(p, text, signal) : undefined));
|
|
792
|
+
if (!chosen)
|
|
793
|
+
return undefined;
|
|
794
|
+
return viaPlayer(chosen) ? synthesizeWith(chosen, text, signal) : plainSpeech(chosen, signal);
|
|
795
|
+
};
|
|
796
|
+
const synthesizeWith = async (first, text, signal) => {
|
|
797
|
+
let provider = first;
|
|
798
|
+
while (provider && !signal.aborted) {
|
|
799
|
+
if (!viaPlayer(provider))
|
|
800
|
+
return plainSpeech(provider, signal);
|
|
801
|
+
const plan = planProsody(provider.prosody, effects?.prosody?.(), true);
|
|
802
|
+
const speakOptions = { language, voice: options.voiceName, signal, prosody: plan.synth };
|
|
803
|
+
try {
|
|
804
|
+
const audio = await synthesize(provider, text, speakOptions, signal);
|
|
805
|
+
return audio && { provider, options: speakOptions, plan, audio };
|
|
806
|
+
}
|
|
807
|
+
catch (error) {
|
|
808
|
+
if (signal.aborted)
|
|
809
|
+
return undefined;
|
|
810
|
+
// This sentence goes to the next ready provider (the backend is down, the model choked).
|
|
811
|
+
const failed = provider;
|
|
812
|
+
provider = ttsProviders.slice(ttsProviders.indexOf(failed) + 1).find((p) => usable(p) && ready(p));
|
|
813
|
+
// The last failure is reported once, by the chunk, under this provider's name.
|
|
814
|
+
if (!provider)
|
|
815
|
+
throw new ProviderFailure(failed.name, error);
|
|
816
|
+
ctx.emit('error', { error, source: `voice:tts:${failed.name}` });
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
return undefined;
|
|
820
|
+
};
|
|
821
|
+
const willSpeakReplies = () => state.output === 'voice' || (state.output === 'auto' && state.lastInput === 'voice');
|
|
822
|
+
const speakChunk = (text, onSpoken, at) => {
|
|
823
|
+
if (!preferred() || !text.trim()) {
|
|
824
|
+
if (at && text.trim())
|
|
825
|
+
ctx.emit('voice.speech', { phase: 'skip', reason: 'no-provider', ...at });
|
|
826
|
+
onSpoken?.();
|
|
827
|
+
return Promise.resolve();
|
|
828
|
+
}
|
|
829
|
+
const controller = (speechController ??= new AbortController());
|
|
830
|
+
chunksLeft++;
|
|
831
|
+
let job;
|
|
832
|
+
try {
|
|
833
|
+
job = prepare(text, controller.signal);
|
|
834
|
+
}
|
|
835
|
+
catch (error) {
|
|
836
|
+
job = Promise.reject(error);
|
|
837
|
+
}
|
|
838
|
+
if (job instanceof Promise)
|
|
839
|
+
job.catch(() => { });
|
|
840
|
+
speechQueue = speechQueue.then(async () => {
|
|
841
|
+
let prepared;
|
|
842
|
+
let sentence;
|
|
843
|
+
try {
|
|
844
|
+
prepared = job instanceof Promise ? await job : job;
|
|
845
|
+
if (controller.signal.aborted) {
|
|
846
|
+
if (at)
|
|
847
|
+
ctx.emit('voice.speech', { phase: 'skip', reason: 'cancelled', ...at });
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
// Nobody can speak it yet: the text stays as shown.
|
|
851
|
+
if (!prepared) {
|
|
852
|
+
if (at)
|
|
853
|
+
ctx.emit('voice.speech', { phase: 'skip', reason: 'no-provider', ...at });
|
|
854
|
+
return onSpoken?.();
|
|
855
|
+
}
|
|
856
|
+
set({ speaking: true, tts: prepared.provider.name });
|
|
857
|
+
ctx.emit('voice.speech', { phase: 'start', provider: prepared.provider.name, chars: text.length, ...at });
|
|
858
|
+
sentence = { text, startedAt: Date.now() };
|
|
859
|
+
recentSpeech().push(sentence);
|
|
860
|
+
// Without barge-in the mic would hear the bot: pause recognition while speaking.
|
|
861
|
+
if (!bargeIn && active?.session) {
|
|
862
|
+
pausedForSpeech = true;
|
|
863
|
+
active.session.abort();
|
|
864
|
+
}
|
|
865
|
+
if (prepared.audio) {
|
|
866
|
+
await player.play(prepared.audio, {
|
|
867
|
+
effects,
|
|
868
|
+
playbackRate: prepared.plan.playbackRate,
|
|
869
|
+
gainDb: prepared.plan.gainDb,
|
|
870
|
+
signal: controller.signal,
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
else {
|
|
874
|
+
await prepared.provider.speak(text, prepared.options);
|
|
875
|
+
}
|
|
876
|
+
const interrupted = controller.signal.aborted;
|
|
877
|
+
ctx.emit('voice.speech', {
|
|
878
|
+
phase: 'end',
|
|
879
|
+
provider: prepared.provider.name,
|
|
880
|
+
ms: Date.now() - sentence.startedAt,
|
|
881
|
+
...(interrupted ? { interrupted } : {}),
|
|
882
|
+
...at,
|
|
883
|
+
});
|
|
884
|
+
if (!interrupted)
|
|
885
|
+
onSpoken?.();
|
|
886
|
+
}
|
|
887
|
+
catch (error) {
|
|
888
|
+
if (!controller.signal.aborted) {
|
|
889
|
+
const failure = error instanceof ProviderFailure ? error : undefined;
|
|
890
|
+
const speechError = speakError(failure ? failure.error : error);
|
|
891
|
+
const provider = failure?.provider ?? prepared?.provider.name;
|
|
892
|
+
ctx.emit('voice.speech', { phase: 'error', provider, code: speechError.code, message: speechError.message, ...at });
|
|
893
|
+
fail(speechError, {}, provider ? `voice:tts:${provider}` : 'voice');
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
finally {
|
|
897
|
+
if (sentence)
|
|
898
|
+
sentence.endedAt ??= Date.now();
|
|
899
|
+
chunksLeft = Math.max(0, chunksLeft - 1);
|
|
900
|
+
if (chunksLeft === 0 || controller.signal.aborted)
|
|
901
|
+
set({ speaking: false });
|
|
902
|
+
if (pausedForSpeech) {
|
|
903
|
+
pausedForSpeech = false;
|
|
904
|
+
if (wantListening && state.mode === 'hands-free')
|
|
905
|
+
void startSession();
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
});
|
|
909
|
+
return speechQueue;
|
|
910
|
+
};
|
|
911
|
+
/** Speaks the complete sentences of an assistant message as they stream in (`end`: the rest too). */
|
|
912
|
+
const follow = (messageId, end) => {
|
|
913
|
+
const message = ctx.app.conversation.find(messageId);
|
|
914
|
+
if (message?.role !== 'assistant')
|
|
915
|
+
return;
|
|
916
|
+
let track = tracks.get(messageId);
|
|
917
|
+
if (!track) {
|
|
918
|
+
const speak = willSpeakReplies();
|
|
919
|
+
track = { speak, queued: 0, spoken: 0 };
|
|
920
|
+
tracks.set(messageId, track);
|
|
921
|
+
if (tracks.size > 8)
|
|
922
|
+
tracks.delete(tracks.keys().next().value);
|
|
923
|
+
if (state.interrupted)
|
|
924
|
+
set({ interrupted: false });
|
|
925
|
+
ctx.emit('voice.track', { messageId, speak });
|
|
926
|
+
}
|
|
927
|
+
if (!track.speak || message.interrupted)
|
|
928
|
+
return;
|
|
929
|
+
const text = textOf(message);
|
|
930
|
+
const current = track;
|
|
931
|
+
// One chunk per sentence, so an interruption knows which sentences were heard.
|
|
932
|
+
while (current.queued < text.length) {
|
|
933
|
+
const sentence = sentenceEnd(text, current.queued);
|
|
934
|
+
const until = sentence > current.queued ? sentence : end ? text.length : -1;
|
|
935
|
+
if (until < 0)
|
|
936
|
+
return;
|
|
937
|
+
const chunk = text.slice(current.queued, until);
|
|
938
|
+
current.queued = until;
|
|
939
|
+
void speakChunk(chunk, () => (current.spoken = until), { messageId, from: until - chunk.length, to: until });
|
|
940
|
+
}
|
|
941
|
+
};
|
|
942
|
+
/* ---------------------------------------------------------------------- */
|
|
943
|
+
/* Service */
|
|
944
|
+
/* ---------------------------------------------------------------------- */
|
|
945
|
+
const service = {
|
|
946
|
+
getState: () => state,
|
|
947
|
+
subscribe(listener) {
|
|
948
|
+
listeners.add(listener);
|
|
949
|
+
return () => listeners.delete(listener);
|
|
950
|
+
},
|
|
951
|
+
async listen(options = {}) {
|
|
952
|
+
if (active && !active.stopping)
|
|
953
|
+
return;
|
|
954
|
+
if (active) {
|
|
955
|
+
// Pressed again while the previous session was still flushing.
|
|
956
|
+
const previous = active;
|
|
957
|
+
active = undefined;
|
|
958
|
+
clearTimeout(previous.killTimer);
|
|
959
|
+
previous.session?.abort();
|
|
960
|
+
if (previous.held)
|
|
961
|
+
release(previous.held);
|
|
962
|
+
}
|
|
963
|
+
clearTimeout(restartTimer);
|
|
964
|
+
if (holding)
|
|
965
|
+
release(holding);
|
|
966
|
+
holding =
|
|
967
|
+
options.until === 'stop' && state.mode === 'push-to-talk' ? { texts: [], done: false } : undefined;
|
|
968
|
+
wantListening = true;
|
|
969
|
+
// Usually a click: the reply to come may be played long after it.
|
|
970
|
+
unlockAudio();
|
|
971
|
+
// Pressing to talk while the bot answers is a barge-in.
|
|
972
|
+
if (bargeIn && botAnswering()) {
|
|
973
|
+
bargedIn = true;
|
|
974
|
+
decide({ kind: 'interrupt', text: '' });
|
|
975
|
+
void ctx.app.interrupt({ by: 'voice' });
|
|
976
|
+
service.cancelSpeech();
|
|
977
|
+
set({ interrupted: true });
|
|
978
|
+
}
|
|
979
|
+
else if (state.speaking) {
|
|
980
|
+
service.cancelSpeech();
|
|
981
|
+
}
|
|
982
|
+
await startSession();
|
|
983
|
+
},
|
|
984
|
+
stop() {
|
|
985
|
+
wantListening = false;
|
|
986
|
+
clearTimeout(restartTimer);
|
|
987
|
+
const entry = active;
|
|
988
|
+
const press = holding;
|
|
989
|
+
holding = undefined;
|
|
990
|
+
// The mic turns off for the user right away; the provider flushes in the background.
|
|
991
|
+
set({ listening: false, partial: state.mode === 'hands-free' ? turnText('') : '' });
|
|
992
|
+
if (!entry || entry.stopping) {
|
|
993
|
+
// Released between two sessions of a hold-to-talk press.
|
|
994
|
+
if (press)
|
|
995
|
+
release(press);
|
|
996
|
+
if (turn.texts.length > 0)
|
|
997
|
+
sendTurn('manual');
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
entry.stopping = true;
|
|
1001
|
+
entry.session?.stop();
|
|
1002
|
+
entry.killTimer = setTimeout(() => {
|
|
1003
|
+
if (active === entry) {
|
|
1004
|
+
entry.session?.abort();
|
|
1005
|
+
active = undefined;
|
|
1006
|
+
}
|
|
1007
|
+
if (entry.held)
|
|
1008
|
+
release(entry.held);
|
|
1009
|
+
if (turn.texts.length > 0)
|
|
1010
|
+
sendTurn('manual');
|
|
1011
|
+
}, 3000);
|
|
1012
|
+
},
|
|
1013
|
+
cancel() {
|
|
1014
|
+
wantListening = false;
|
|
1015
|
+
clearTimeout(restartTimer);
|
|
1016
|
+
if (holding)
|
|
1017
|
+
release(holding, false);
|
|
1018
|
+
discardPending();
|
|
1019
|
+
const entry = active;
|
|
1020
|
+
active = undefined;
|
|
1021
|
+
set({ listening: false, partial: '' });
|
|
1022
|
+
if (!entry)
|
|
1023
|
+
return;
|
|
1024
|
+
entry.discarded = true;
|
|
1025
|
+
if (entry.held)
|
|
1026
|
+
release(entry.held, false);
|
|
1027
|
+
clearTimeout(entry.killTimer);
|
|
1028
|
+
entry.session?.abort();
|
|
1029
|
+
},
|
|
1030
|
+
endTurn() {
|
|
1031
|
+
if (turn.texts.length > 0 || turn.live) {
|
|
1032
|
+
// Words still being recognized go out too.
|
|
1033
|
+
sentLive = normalize(turn.live);
|
|
1034
|
+
turn.texts.push(...(turn.live ? [turn.live] : []));
|
|
1035
|
+
turn.live = '';
|
|
1036
|
+
sendTurn('manual');
|
|
1037
|
+
}
|
|
1038
|
+
},
|
|
1039
|
+
discardTurn() {
|
|
1040
|
+
discardPending();
|
|
1041
|
+
if (state.mode !== 'hands-free')
|
|
1042
|
+
return;
|
|
1043
|
+
// Drop what the recognizer is still holding as well.
|
|
1044
|
+
const entry = active;
|
|
1045
|
+
if (entry && !entry.stopping) {
|
|
1046
|
+
entry.discarded = true;
|
|
1047
|
+
active = undefined;
|
|
1048
|
+
entry.session?.abort();
|
|
1049
|
+
if (wantListening)
|
|
1050
|
+
restartTimer = setTimeout(() => wantListening && !active && void startSession(), 250);
|
|
1051
|
+
}
|
|
1052
|
+
},
|
|
1053
|
+
async toggle() {
|
|
1054
|
+
if (state.listening || wantListening)
|
|
1055
|
+
service.stop();
|
|
1056
|
+
else
|
|
1057
|
+
await service.listen();
|
|
1058
|
+
},
|
|
1059
|
+
setMode(mode) {
|
|
1060
|
+
if (mode === state.mode)
|
|
1061
|
+
return;
|
|
1062
|
+
const wasListening = wantListening;
|
|
1063
|
+
service.stop();
|
|
1064
|
+
set({ mode });
|
|
1065
|
+
if (wasListening && mode === 'hands-free')
|
|
1066
|
+
void service.listen();
|
|
1067
|
+
},
|
|
1068
|
+
setOutput(output) {
|
|
1069
|
+
set({ output });
|
|
1070
|
+
// The user turned the bot's voice on (the widget's speaker button sets `auto`).
|
|
1071
|
+
if (output !== 'text' && loadStart === 'first-use')
|
|
1072
|
+
void loadPreferred();
|
|
1073
|
+
if (output === 'text')
|
|
1074
|
+
service.cancelSpeech();
|
|
1075
|
+
},
|
|
1076
|
+
setEndOfTurn(provider) {
|
|
1077
|
+
void setEndOfTurn(provider);
|
|
1078
|
+
},
|
|
1079
|
+
setEcho(filter) {
|
|
1080
|
+
echo = filter ?? undefined;
|
|
1081
|
+
set({ echo: echo?.name });
|
|
1082
|
+
},
|
|
1083
|
+
configure(change) {
|
|
1084
|
+
if (change.turn)
|
|
1085
|
+
timing = { ...timing, ...change.turn };
|
|
1086
|
+
if (change.bargeIn !== undefined) {
|
|
1087
|
+
bargeIn = change.bargeIn !== false;
|
|
1088
|
+
if (typeof change.bargeIn === 'object')
|
|
1089
|
+
bargeInOptions = { ...bargeInOptions, ...change.bargeIn };
|
|
1090
|
+
}
|
|
1091
|
+
},
|
|
1092
|
+
speak(text) {
|
|
1093
|
+
return speakChunk(text);
|
|
1094
|
+
},
|
|
1095
|
+
cancelSpeech() {
|
|
1096
|
+
speechController?.abort();
|
|
1097
|
+
speechController = undefined;
|
|
1098
|
+
player.stop();
|
|
1099
|
+
for (const p of ttsProviders)
|
|
1100
|
+
p.cancel?.();
|
|
1101
|
+
speechQueue = Promise.resolve();
|
|
1102
|
+
chunksLeft = 0;
|
|
1103
|
+
// What was playing can still reach the recognizer for a moment.
|
|
1104
|
+
const now = Date.now();
|
|
1105
|
+
for (const sentence of speech)
|
|
1106
|
+
sentence.endedAt ??= now;
|
|
1107
|
+
// Messages already queued stop being spoken; their `spoken` count stays.
|
|
1108
|
+
for (const track of tracks.values())
|
|
1109
|
+
track.speak = false;
|
|
1110
|
+
if (state.speaking)
|
|
1111
|
+
set({ speaking: false });
|
|
1112
|
+
},
|
|
1113
|
+
loadSpeech(name) {
|
|
1114
|
+
if (name === undefined)
|
|
1115
|
+
return loadPreferred();
|
|
1116
|
+
const p = ttsProviders.find((provider) => provider.name === name);
|
|
1117
|
+
return p ? loadProvider(p) : Promise.resolve(false);
|
|
1118
|
+
},
|
|
1119
|
+
async setTTS(tts) {
|
|
1120
|
+
const next = (await supportedOf(toArray(tts))).filter(playable);
|
|
1121
|
+
service.cancelSpeech();
|
|
1122
|
+
for (const p of ttsProviders)
|
|
1123
|
+
if (!next.includes(p))
|
|
1124
|
+
statuses.delete(p);
|
|
1125
|
+
ttsProviders = next;
|
|
1126
|
+
followTraces(next);
|
|
1127
|
+
set({ supported: { ...state.supported, tts: next.length > 0 }, tts: next[0]?.name });
|
|
1128
|
+
publishStatus();
|
|
1129
|
+
if (loadingWanted || loadStart === 'eager')
|
|
1130
|
+
void loadPreferred();
|
|
1131
|
+
},
|
|
1132
|
+
setEffects(next) {
|
|
1133
|
+
effects = next ?? undefined;
|
|
1134
|
+
followEffects();
|
|
1135
|
+
},
|
|
1136
|
+
unlockAudio,
|
|
1137
|
+
willSpeakReplies,
|
|
1138
|
+
};
|
|
1139
|
+
publishStatus();
|
|
1140
|
+
followEffects();
|
|
1141
|
+
// Which modality did the user use last?
|
|
1142
|
+
ctx.on('signal', (signal) => {
|
|
1143
|
+
if (signal.kind !== 'trigger')
|
|
1144
|
+
return;
|
|
1145
|
+
if (signal.modality === 'transcript' || signal.modality === 'audio')
|
|
1146
|
+
set({ lastInput: 'voice' });
|
|
1147
|
+
else if (signal.modality === 'text')
|
|
1148
|
+
set({ lastInput: 'keyboard' });
|
|
1149
|
+
});
|
|
1150
|
+
// Output follows input: speak assistant replies sentence by sentence as they arrive.
|
|
1151
|
+
ctx.on('stimulus', ({ stimulus }) => {
|
|
1152
|
+
if (stimulus.type === 'message.delta' || stimulus.type === 'message.part')
|
|
1153
|
+
follow(stimulus.messageId, false);
|
|
1154
|
+
else if (stimulus.type === 'message.end')
|
|
1155
|
+
follow(stimulus.messageId, true);
|
|
1156
|
+
});
|
|
1157
|
+
// An interrupted reply keeps only what was spoken.
|
|
1158
|
+
ctx.intercept('interrupt.before', (point) => {
|
|
1159
|
+
const track = tracks.get(point.messageId);
|
|
1160
|
+
if (!track || track.queued === 0)
|
|
1161
|
+
return;
|
|
1162
|
+
if (track.spoken >= point.text.length)
|
|
1163
|
+
return;
|
|
1164
|
+
return { ...point, keep: Math.min(point.keep, track.spoken) };
|
|
1165
|
+
});
|
|
1166
|
+
// A new turn from the keyboard makes pending speech obsolete.
|
|
1167
|
+
ctx.on('turn.status', (status) => {
|
|
1168
|
+
if (status.phase === 'received' && status.signal?.modality === 'text')
|
|
1169
|
+
service.cancelSpeech();
|
|
1170
|
+
});
|
|
1171
|
+
ctx.onDispose(() => {
|
|
1172
|
+
disposed = true;
|
|
1173
|
+
unfollowEffects?.();
|
|
1174
|
+
followTraces([]);
|
|
1175
|
+
for (const controller of loadControllers)
|
|
1176
|
+
controller.abort();
|
|
1177
|
+
if (typeof document !== 'undefined') {
|
|
1178
|
+
for (const type of gestures)
|
|
1179
|
+
document.removeEventListener(type, unlockAudio, { capture: true });
|
|
1180
|
+
}
|
|
1181
|
+
wantListening = false;
|
|
1182
|
+
if (holding)
|
|
1183
|
+
release(holding, false);
|
|
1184
|
+
clearTimeout(restartTimer);
|
|
1185
|
+
clearTurnTimers();
|
|
1186
|
+
active?.session?.abort();
|
|
1187
|
+
service.cancelSpeech();
|
|
1188
|
+
listeners.clear();
|
|
1189
|
+
});
|
|
1190
|
+
return service;
|
|
1191
|
+
}
|
|
1192
|
+
/** The last provider that failed a sentence, so the error is reported once and under its name. */
|
|
1193
|
+
class ProviderFailure {
|
|
1194
|
+
provider;
|
|
1195
|
+
error;
|
|
1196
|
+
constructor(provider, error) {
|
|
1197
|
+
this.provider = provider;
|
|
1198
|
+
this.error = error;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
/** Errors while speaking always carry `stage: 'speak'`, so UIs don't blame the microphone. */
|
|
1202
|
+
function speakError(error) {
|
|
1203
|
+
if (error instanceof SpeechError)
|
|
1204
|
+
return error.stage === 'speak' ? error : new SpeechError(error.code, error.message, 'speak');
|
|
1205
|
+
const name = error?.name;
|
|
1206
|
+
const message = error?.message ?? String(error);
|
|
1207
|
+
return new SpeechError(name === 'NotAllowedError' ? 'output-blocked' : 'synthesis-failed', message, 'speak');
|
|
1208
|
+
}
|
|
1209
|
+
//# sourceMappingURL=voice.js.map
|