dsh-live-voice 0.0.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +1 -1
- package/lib/client.js +795 -207
- package/lib/server.js +18 -3
- package/package.json +2 -1
- package/scripts/build.ts +1 -1
- package/src/client/chat.ts +7 -0
- package/src/client/components.ts +453 -190
- package/src/client/index.ts +111 -11
- package/src/client/qwen-settings.ts +1 -1
- package/src/client/styles.ts +11 -7
- package/src/core/coordinator.ts +209 -26
- package/src/core/filters.ts +63 -0
- package/src/core/microphone.ts +7 -1
- package/src/core/ownership.ts +1 -1
- package/src/core/settings.ts +102 -3
- package/src/engines/qwen-http-host.ts +3 -9
- package/src/engines/recognition/browser.ts +23 -0
- package/src/engines/speaking/qwen-http.ts +6 -1
package/src/client/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// @ts-nocheck
|
|
2
2
|
import React from 'react';
|
|
3
|
+
import { createPortal } from 'react-dom';
|
|
3
4
|
import { VoiceCoordinator } from '../core/coordinator.ts';
|
|
4
5
|
import { VoiceOwnership } from '../core/ownership.ts';
|
|
5
6
|
import { normalizeSettings } from '../core/settings.ts';
|
|
@@ -12,15 +13,19 @@ import { QwenHttpRecognitionEngine } from '../engines/recognition/qwen-http.ts';
|
|
|
12
13
|
import { QwenHttpSpeakingEngine } from '../engines/speaking/qwen-http.ts';
|
|
13
14
|
import { createComponents } from './components.ts';
|
|
14
15
|
import { styles } from './styles.ts';
|
|
15
|
-
import { assistantMessages, addressedTurn, latestUserSequence } from './chat.ts';
|
|
16
|
+
import { assistantMessages, addressedTurn, latestUserSequence, pendingQuestionSpeech } from './chat.ts';
|
|
16
17
|
|
|
17
|
-
export const inject = ['slots', 'connection', 'uiConversation'];
|
|
18
|
+
export const inject = ['slots', 'connection', 'uiConversation', 'uiSession'];
|
|
18
19
|
export function apply(ctx) {
|
|
19
20
|
const e = React.createElement;
|
|
20
21
|
const { MicrophoneButtons, RecordingBar, SpeakButton, SettingsPanel } = createComponents(React);
|
|
21
22
|
const controllers = new Map();
|
|
22
23
|
const retiring = new Set();
|
|
23
24
|
let disposed = false;
|
|
25
|
+
// Voice conversation mode belongs to the plugin, not to a mounted composer.
|
|
26
|
+
// A route change may replace every conversation slot, but the next committed
|
|
27
|
+
// composer should inherit the user's explicit choice to remain in voice mode.
|
|
28
|
+
let voiceModeActive = false;
|
|
24
29
|
const ownership = new VoiceOwnership();
|
|
25
30
|
const recognitionFor = (settings, meter) =>
|
|
26
31
|
settings.recognitionEngine === 'qwen-http'
|
|
@@ -49,6 +54,10 @@ export function apply(ctx) {
|
|
|
49
54
|
entry.composers.clear();
|
|
50
55
|
entry.unsubscribe?.();
|
|
51
56
|
entry.unsubscribe = null;
|
|
57
|
+
entry.unsubscribePendingQuestion?.();
|
|
58
|
+
entry.unsubscribePendingQuestion = null;
|
|
59
|
+
entry.questionCapture = null;
|
|
60
|
+
entry.controller.patch({ answeringQuestion: false });
|
|
52
61
|
entry.chatListeners.clear();
|
|
53
62
|
if (controllers.get(entry.key) === entry) controllers.delete(entry.key);
|
|
54
63
|
// Keep teardown in the hardware handoff barrier, not in the session registry.
|
|
@@ -89,9 +98,32 @@ export function apply(ctx) {
|
|
|
89
98
|
meter,
|
|
90
99
|
composer: {
|
|
91
100
|
getDraft: () => entry.draft,
|
|
92
|
-
submit: () => {
|
|
101
|
+
submit: (mode = 'queue') => {
|
|
93
102
|
const owner = [...entry.composers.values()].at(-1);
|
|
94
|
-
owner?.actions.submit?.();
|
|
103
|
+
owner?.actions.submit?.(mode);
|
|
104
|
+
},
|
|
105
|
+
handleQuestionResult: ({ final, interim }) => {
|
|
106
|
+
const capture = entry.questionCapture;
|
|
107
|
+
if (!capture || capture.answering || (!final && !interim)) return false;
|
|
108
|
+
if (final?.trim()) {
|
|
109
|
+
capture.answering = true;
|
|
110
|
+
capture.answers.push({
|
|
111
|
+
id: capture.interaction.questions[capture.index].id,
|
|
112
|
+
selected: [],
|
|
113
|
+
custom: final.trim(),
|
|
114
|
+
});
|
|
115
|
+
capture.index++;
|
|
116
|
+
if (capture.index < capture.interaction.questions.length) {
|
|
117
|
+
capture.answering = false;
|
|
118
|
+
const text = pendingQuestionSpeech(capture.interaction, capture.index);
|
|
119
|
+
if (text) run(entry.controller, entry.controller.speak(text, capture.interaction.key));
|
|
120
|
+
} else {
|
|
121
|
+
entry.questionCapture = null;
|
|
122
|
+
entry.controller.patch({ answeringQuestion: false });
|
|
123
|
+
run(entry.controller, capture.interaction.answer({ answers: capture.answers }));
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return true;
|
|
95
127
|
},
|
|
96
128
|
setDraft: (text) => {
|
|
97
129
|
if (disposed || entry.closed) return;
|
|
@@ -140,6 +172,30 @@ export function apply(ctx) {
|
|
|
140
172
|
refresh();
|
|
141
173
|
for (const listener of entry.chatListeners) listener();
|
|
142
174
|
});
|
|
175
|
+
const pendingInteractions = ctx.uiSession.pendingInteractions;
|
|
176
|
+
const refreshPendingQuestion = (baseline = false) => {
|
|
177
|
+
if (disposed || entry.closed) return;
|
|
178
|
+
const interaction = pendingInteractions.getSnapshot().get(sessionId);
|
|
179
|
+
const key = interaction?.kind === 'question' ? interaction.key : null;
|
|
180
|
+
if (!key || (entry.questionCapture && entry.questionCapture.interaction.key !== key)) {
|
|
181
|
+
entry.questionCapture = null;
|
|
182
|
+
controller.patch({ answeringQuestion: false });
|
|
183
|
+
}
|
|
184
|
+
if (baseline) {
|
|
185
|
+
entry.pendingQuestionKey = key;
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
if (!key || key === entry.pendingQuestionKey) return;
|
|
189
|
+
entry.pendingQuestionKey = key;
|
|
190
|
+
const text = pendingQuestionSpeech(interaction);
|
|
191
|
+
if (text && controller.getSnapshot().conversation) {
|
|
192
|
+
entry.questionCapture = { interaction, index: 0, answers: [], answering: false };
|
|
193
|
+
controller.patch({ answeringQuestion: true });
|
|
194
|
+
run(controller, controller.speak(text, key));
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
refreshPendingQuestion(true);
|
|
198
|
+
entry.unsubscribePendingQuestion = pendingInteractions.subscribe(refreshPendingQuestion);
|
|
143
199
|
const update = controller.updateSettings.bind(controller);
|
|
144
200
|
controller.updateSettings = (next) => {
|
|
145
201
|
if (disposed || entry.closed) return;
|
|
@@ -160,6 +216,7 @@ export function apply(ctx) {
|
|
|
160
216
|
const original = controller[method].bind(controller);
|
|
161
217
|
controller[method] = (...args) => {
|
|
162
218
|
entry.request++;
|
|
219
|
+
if (method === 'endConversation' && args[0] !== true) voiceModeActive = false;
|
|
163
220
|
return original(...args);
|
|
164
221
|
};
|
|
165
222
|
}
|
|
@@ -168,6 +225,7 @@ export function apply(ctx) {
|
|
|
168
225
|
controller[method] = (...args) => {
|
|
169
226
|
if (disposed || entry.closed || !entry.refs) return Promise.resolve();
|
|
170
227
|
if (method !== 'speak' && !entry.composers.size) return Promise.resolve();
|
|
228
|
+
if (method === 'startConversation') voiceModeActive = true;
|
|
171
229
|
const request = ++entry.request;
|
|
172
230
|
return ownership.run(
|
|
173
231
|
controller,
|
|
@@ -218,19 +276,17 @@ export function apply(ctx) {
|
|
|
218
276
|
return () => {
|
|
219
277
|
entry.composers.delete(token.current);
|
|
220
278
|
// Message-action mounts may outlive the composer; never retain its actions.
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
if (!entry.closed && !entry.composers.size)
|
|
225
|
-
run(entry.controller, entry.controller.endConversation());
|
|
226
|
-
});
|
|
227
|
-
}
|
|
279
|
+
// Do not interpret a route-driven composer unmount as the user's request
|
|
280
|
+
// to leave voice mode. Retirement still releases the old controller; the
|
|
281
|
+
// next committed composer resumes the conversation automatically.
|
|
228
282
|
};
|
|
229
283
|
}, [entry]);
|
|
230
284
|
React.useLayoutEffect(() => {
|
|
231
285
|
if (!entry || entry.closed || disposed) return;
|
|
232
286
|
if (!input || typeof props.inputActions?.setDraft !== 'function') return;
|
|
233
287
|
entry.composers.set(token.current, { actions: props.inputActions });
|
|
288
|
+
if (voiceModeActive && !entry.controller.getSnapshot().conversation)
|
|
289
|
+
run(entry.controller, entry.controller.startConversation());
|
|
234
290
|
}, [entry, input, props.inputActions]);
|
|
235
291
|
React.useLayoutEffect(() => {
|
|
236
292
|
if (!entry || entry.closed || disposed || !input) return;
|
|
@@ -334,6 +390,49 @@ export function apply(ctx) {
|
|
|
334
390
|
useComposer(entry, props);
|
|
335
391
|
return entry ? e(RecordingBar, { controller: entry.controller }) : null;
|
|
336
392
|
}
|
|
393
|
+
function QuestionStatusView({ entry }) {
|
|
394
|
+
const snapshot = React.useSyncExternalStore(
|
|
395
|
+
entry.controller.subscribe,
|
|
396
|
+
entry.controller.getSnapshot,
|
|
397
|
+
);
|
|
398
|
+
const [overlayStyle, setOverlayStyle] = React.useState();
|
|
399
|
+
React.useLayoutEffect(() => {
|
|
400
|
+
if (!snapshot.answeringQuestion) return;
|
|
401
|
+
const seat = document.querySelector('[data-composer-seat]');
|
|
402
|
+
if (!seat) return;
|
|
403
|
+
const update = () => {
|
|
404
|
+
const rect = seat.getBoundingClientRect();
|
|
405
|
+
setOverlayStyle({
|
|
406
|
+
left: rect.left + rect.width / 2,
|
|
407
|
+
width: Math.min(720, Math.max(0, rect.width - 32)),
|
|
408
|
+
});
|
|
409
|
+
};
|
|
410
|
+
update();
|
|
411
|
+
const observer = typeof ResizeObserver === 'function' ? new ResizeObserver(update) : null;
|
|
412
|
+
observer?.observe(seat);
|
|
413
|
+
window.addEventListener('resize', update);
|
|
414
|
+
return () => {
|
|
415
|
+
observer?.disconnect();
|
|
416
|
+
window.removeEventListener('resize', update);
|
|
417
|
+
};
|
|
418
|
+
}, [snapshot.answeringQuestion]);
|
|
419
|
+
const target = typeof document === 'undefined' ? null : document.body;
|
|
420
|
+
return snapshot.answeringQuestion && target && overlayStyle
|
|
421
|
+
? createPortal(
|
|
422
|
+
e(RecordingBar, {
|
|
423
|
+
controller: entry.controller,
|
|
424
|
+
questionOnly: true,
|
|
425
|
+
overlay: true,
|
|
426
|
+
overlayStyle,
|
|
427
|
+
}),
|
|
428
|
+
target,
|
|
429
|
+
)
|
|
430
|
+
: null;
|
|
431
|
+
}
|
|
432
|
+
function QuestionStatus(props) {
|
|
433
|
+
const entry = useEntry(props.sessionId, 'question-status');
|
|
434
|
+
return entry ? e(QuestionStatusView, { entry }) : null;
|
|
435
|
+
}
|
|
337
436
|
function ActionView({ entry, messageId }) {
|
|
338
437
|
const snapshot = React.useSyncExternalStore(
|
|
339
438
|
entry.controller.subscribe,
|
|
@@ -369,6 +468,7 @@ export function apply(ctx) {
|
|
|
369
468
|
for (const [name, id, order, component] of [
|
|
370
469
|
['conversation.input.right', 'live-voice-controls', 6, Buttons],
|
|
371
470
|
['conversation.input.dock', 'live-voice-status', -100, Dock],
|
|
471
|
+
['conversation.session.header.utilities', 'live-voice-question-status', 100, QuestionStatus],
|
|
372
472
|
['conversation.chat.assistant-actions', 'live-voice-speak', 5, Action],
|
|
373
473
|
])
|
|
374
474
|
ctx.slots.inject(name, () =>
|
|
@@ -113,7 +113,7 @@ export function createQwenSettings(React) {
|
|
|
113
113
|
h(
|
|
114
114
|
'p',
|
|
115
115
|
null,
|
|
116
|
-
'Host-wide settings for the
|
|
116
|
+
'Host-wide settings for the Qwen3 ASR + TTS server. Enter any HTTP or HTTPS base URL reachable from the DSH host. The browser accesses it through authenticated DSH routes.',
|
|
117
117
|
),
|
|
118
118
|
field('Qwen API base URL', 'baseUrl'),
|
|
119
119
|
field('Request timeout (ms)', 'timeoutMs', 'number'),
|
package/src/client/styles.ts
CHANGED
|
@@ -11,23 +11,27 @@ export const styles = `
|
|
|
11
11
|
.dlv-icon-button:disabled{opacity:.4;cursor:default}
|
|
12
12
|
.dlv-icon-button:focus-visible,.dlv-settings :is(input,select):focus-visible{outline:2px solid var(--dsw-alias-label-primary);outline-offset:3px}
|
|
13
13
|
.dlv-bar-wrap{width:100%;min-width:0}
|
|
14
|
+
.dlv-question-overlay{position:fixed;z-index:10000;bottom:8px;transform:translateX(-50%);max-width:calc(100vw - 32px);pointer-events:none}
|
|
15
|
+
.dlv-question-overlay .dlv-pill{pointer-events:auto}
|
|
14
16
|
.dlv-pill{display:flex;align-items:center;box-sizing:border-box;gap:10px;min-height:52px;border-radius:26px;background:var(--dsw-alias-bg-layer-1);border:1px solid var(--dsw-alias-border-l1);padding:0 14px;width:100%;max-width:720px;margin:0 auto;box-shadow:0 8px 24px rgba(0,0,0,.18)}
|
|
15
17
|
.dlv-pill-button{width:34px;height:34px;border-radius:50%;border:1px solid var(--dsw-alias-border-l2);color:var(--dsw-alias-label-primary)}
|
|
16
18
|
.dlv-pill-button:hover{background:var(--dsw-alias-bg-layer-2)}
|
|
17
|
-
.dlv-live-toggle{width:
|
|
18
|
-
.dlv-live-toggle svg{width:18px;height:18px}
|
|
19
|
-
.dlv-toggle-state{
|
|
20
|
-
.dlv-live-toggle:hover{background:var(--dsw-alias-bg-layer-2)}
|
|
21
|
-
.dlv-live-toggle
|
|
19
|
+
.dlv-live-toggle{width:34px;height:34px;padding:0 7px;gap:0;overflow:hidden;border:1px solid var(--dsw-alias-border-l2);border-radius:17px;color:var(--dsw-alias-label-primary);transition:width .16s ease,gap .16s ease}
|
|
20
|
+
.dlv-live-toggle svg{width:18px;height:18px;flex:none}
|
|
21
|
+
.dlv-toggle-state{max-width:0;opacity:0;overflow:hidden;white-space:nowrap;font-size:10px;font-weight:700;line-height:1;letter-spacing:.04em;text-align:left;transition:max-width .16s ease,opacity .12s ease}
|
|
22
|
+
.dlv-live-toggle:hover,.dlv-live-toggle:focus-visible,.dlv-live-toggle-expanded{width:78px;gap:5px;background:var(--dsw-alias-bg-layer-2)}
|
|
23
|
+
.dlv-live-toggle:hover .dlv-toggle-state,.dlv-live-toggle:focus-visible .dlv-toggle-state,.dlv-live-toggle-expanded .dlv-toggle-state{max-width:38px;opacity:1}
|
|
24
|
+
.dlv-live-toggle[aria-checked=false],.dlv-live-toggle[data-mode=manual],.dlv-mic-state[data-muted=true]{color:var(--dsw-alias-label-tertiary);border-color:var(--dsw-alias-border-l1);opacity:.55}
|
|
25
|
+
.dlv-mic-state:hover,.dlv-mic-state:focus-visible{width:102px}.dlv-mic-state:hover .dlv-toggle-state,.dlv-mic-state:focus-visible .dlv-toggle-state{max-width:64px}
|
|
22
26
|
.dlv-live-toggle[aria-checked=true]{background:var(--dsw-alias-bg-layer-2)}
|
|
23
27
|
.dlv-wave{display:block;flex:1 1 180px;min-width:30px;width:100%;height:40px;color:var(--dsw-alias-label-primary)}
|
|
24
28
|
.dlv-status{flex:1 1 120px;min-width:0;font-size:13px;line-height:1.4;color:var(--dsw-alias-label-secondary)}
|
|
25
29
|
.dlv-error{color:var(--dsw-alias-state-error-primary);overflow-wrap:anywhere;font-size:13px;max-width:720px;margin:8px auto}
|
|
26
30
|
.dlv-settings{box-sizing:border-box;padding:18px;width:100%;display:grid;gap:16px;max-width:640px;color:var(--dsw-alias-label-primary)}
|
|
27
31
|
.dlv-settings h3,.dlv-settings p{margin:0}
|
|
28
|
-
.dlv-settings-
|
|
32
|
+
.dlv-settings-tabs{border-bottom:.5px solid var(--dsw-alias-border-l2);display:flex;align-items:flex-end;gap:22px;overflow-x:auto}.dlv-settings-tab{position:relative;flex:none;padding:7px 1px 9px;border:0;background:transparent;color:var(--dsw-alias-label-tertiary);font:inherit;font-size:13px;line-height:20px;cursor:pointer}.dlv-settings-tab:hover,.dlv-settings-tab[data-active=true]{color:var(--dsw-alias-label-primary)}.dlv-settings-tab[data-active=true]:after,.dlv-settings-tab:focus-visible:after{content:"";position:absolute;right:0;bottom:-1px;left:0;height:2px;border-radius:2px 2px 0 0;background:var(--dsw-alias-label-primary)}.dlv-settings-tab:focus-visible{border-radius:2px;outline:2px solid var(--dsw-alias-state-business-primary);outline-offset:2px;color:var(--dsw-alias-label-primary)}.dlv-settings-panel{min-width:0;padding-top:2px}.dlv-settings-panel[hidden]{display:none}.dlv-settings-group{margin:0;border:1px solid var(--dsw-alias-border-l1);border-radius:10px;min-width:0;display:grid;gap:14px;padding:16px}.dlv-settings-group legend{padding:0 6px;font-weight:600;color:var(--dsw-alias-label-primary)}.dlv-settings-card-body{display:grid;gap:12px;padding:12px 0}.dlv-settings-subcard{border:1px solid var(--dsw-alias-border-l2);border-radius:8px;min-width:0}.dlv-settings-subcard>summary{cursor:pointer;font-weight:650;list-style:none;display:flex;align-items:center;justify-content:space-between;padding:10px}.dlv-settings-subcard>summary::-webkit-details-marker{display:none}.dlv-settings-subcard>summary:after{content:"›";transform:rotate(90deg);transition:transform .15s}.dlv-settings-subcard:not([open])>summary:after{transform:rotate(0)}.dlv-settings-subcard-body{display:grid;gap:10px;padding:0 10px 10px}
|
|
29
33
|
.dlv-settings label{display:grid;gap:6px;font-size:14px}.dlv-settings label.dlv-check{display:flex;align-items:center;gap:8px}.dlv-settings label.dlv-check input{width:auto}
|
|
30
|
-
.dlv-settings :is(input,select){box-sizing:border-box;width:100%;padding:8px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;background:var(--dsw-alias-bg-layer-1);color:inherit;font:inherit}
|
|
34
|
+
.dlv-settings :is(input,select,textarea){box-sizing:border-box;width:100%;padding:8px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;background:var(--dsw-alias-bg-layer-1);color:inherit;font:inherit}
|
|
31
35
|
.dlv-settings small,.dlv-settings p{font-size:13px;color:var(--dsw-alias-label-secondary);line-height:1.5}
|
|
32
36
|
.dlv-preset-group{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:8px}.dlv-settings label.dlv-preset{display:flex;align-items:flex-start;gap:8px;padding:10px;border:1px solid var(--dsw-alias-border-l1);border-radius:8px;cursor:pointer}.dlv-settings label.dlv-preset:has(input:checked){border-color:var(--dsw-alias-label-primary);background:var(--dsw-alias-bg-layer-1)}.dlv-settings label.dlv-preset input{width:auto;margin-top:3px}.dlv-preset span{display:grid;gap:3px}.dlv-vad-summary{font-weight:500}
|
|
33
37
|
.dlv-settings-actions{display:flex;flex-wrap:wrap;gap:8px}.dlv-settings-actions button{padding:8px 12px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;background:var(--dsw-alias-bg-layer-1);color:inherit;cursor:pointer}.dlv-settings-actions button:disabled{opacity:.45;cursor:default}
|
package/src/core/coordinator.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
// @ts-nocheck
|
|
2
2
|
import { TranscriptDraft } from './transcript.ts';
|
|
3
3
|
import { normalizeSettings } from './settings.ts';
|
|
4
|
+
import {
|
|
5
|
+
filterSpeechOutput,
|
|
6
|
+
hasMinimumWords,
|
|
7
|
+
hasUnclosedCodeFence,
|
|
8
|
+
matchVoiceCommand,
|
|
9
|
+
splitSpeechOutput,
|
|
10
|
+
} from './filters.ts';
|
|
4
11
|
|
|
5
12
|
const message = (error) => error?.message || String(error);
|
|
6
13
|
function cancellable(promise, signal) {
|
|
@@ -39,20 +46,27 @@ export class VoiceCoordinator {
|
|
|
39
46
|
conversation: false,
|
|
40
47
|
listening: false,
|
|
41
48
|
recognizing: false,
|
|
49
|
+
muted: false,
|
|
42
50
|
speaking: false,
|
|
43
51
|
paused: false,
|
|
44
52
|
starting: false,
|
|
45
53
|
error: null,
|
|
46
54
|
activeMessageId: null,
|
|
55
|
+
answeringQuestion: false,
|
|
47
56
|
autoSendAt: null,
|
|
57
|
+
speechSegmentsRemaining: 0,
|
|
48
58
|
capabilities: {},
|
|
49
59
|
settings: normalizeSettings(settings),
|
|
50
60
|
};
|
|
51
61
|
this.autoSendTimer = null;
|
|
52
62
|
this.autoSendDraft = null;
|
|
63
|
+
this.interruptionTimer = null;
|
|
64
|
+
this.interruptionTranscriptConfirmed = false;
|
|
65
|
+
this.interruptionPausedSpeech = false;
|
|
53
66
|
this.assistantSpeechTimer = null;
|
|
54
67
|
this.assistantSpeechNotBefore = 0;
|
|
55
68
|
this.recognition.lang = this.snapshot.settings.recognitionLang;
|
|
69
|
+
this.meter.deviceId = this.snapshot.settings.inputDeviceId;
|
|
56
70
|
this.getSnapshot = () => this.snapshot;
|
|
57
71
|
this.subscribe = (listener) => {
|
|
58
72
|
if (this.disposed) return () => {};
|
|
@@ -95,6 +109,7 @@ export class VoiceCoordinator {
|
|
|
95
109
|
updateSettings(next) {
|
|
96
110
|
if (this.disposed) return;
|
|
97
111
|
const settings = normalizeSettings({ ...this.snapshot.settings, ...next });
|
|
112
|
+
this.meter.deviceId = settings.inputDeviceId;
|
|
98
113
|
Object.assign(this.recognition, {
|
|
99
114
|
lang: settings.recognitionLang,
|
|
100
115
|
processLocally: settings.recognitionProcessLocally,
|
|
@@ -102,7 +117,7 @@ export class VoiceCoordinator {
|
|
|
102
117
|
voiceDetectionPreset: settings.voiceDetectionPreset,
|
|
103
118
|
});
|
|
104
119
|
this.patch({ settings, error: null });
|
|
105
|
-
if (settings.sendingMode
|
|
120
|
+
if (settings.sendingMode === 'manual') this.cancelAutoSend();
|
|
106
121
|
if (Object.hasOwn(next, 'announceAssistantMessages') && !settings.announceAssistantMessages) {
|
|
107
122
|
this.queue = [];
|
|
108
123
|
this.assistantSpeechNotBefore = 0;
|
|
@@ -169,6 +184,21 @@ export class VoiceCoordinator {
|
|
|
169
184
|
startConversation() {
|
|
170
185
|
return this.startListening(true);
|
|
171
186
|
}
|
|
187
|
+
muteListening() {
|
|
188
|
+
if (!this.snapshot.settings.voiceCommandsEnabled) return this.stopListening();
|
|
189
|
+
this.cancelAutoSend();
|
|
190
|
+
this.composer.setDraft(this.transcript.update(this.composer.getDraft(), '', true));
|
|
191
|
+
this.transcript.reset();
|
|
192
|
+
this.patch({ muted: true, recognizing: false });
|
|
193
|
+
}
|
|
194
|
+
resumeListeningInput() {
|
|
195
|
+
if (this.snapshot.listening || this.snapshot.starting) {
|
|
196
|
+
this.transcript.reset();
|
|
197
|
+
this.patch({ muted: false, recognizing: false });
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
return this.startListening(this.snapshot.conversation);
|
|
201
|
+
}
|
|
172
202
|
startListening(conversation = this.snapshot.conversation) {
|
|
173
203
|
if (this.disposed) return Promise.resolve();
|
|
174
204
|
// Reserve input ownership before awaiting any speech teardown.
|
|
@@ -243,7 +273,7 @@ export class VoiceCoordinator {
|
|
|
243
273
|
this.assistantSpeechNotBefore =
|
|
244
274
|
Date.now() + this.snapshot.settings.assistantSpeechDelaySeconds * 1000;
|
|
245
275
|
this.patch({ recognizing: active });
|
|
246
|
-
|
|
276
|
+
this._handleSpeechInterruption(active);
|
|
247
277
|
if (!active) this._drain();
|
|
248
278
|
},
|
|
249
279
|
onError: (error) => {
|
|
@@ -278,19 +308,134 @@ export class VoiceCoordinator {
|
|
|
278
308
|
}
|
|
279
309
|
});
|
|
280
310
|
}
|
|
311
|
+
_interruptionTranscriptQualifies(text) {
|
|
312
|
+
return hasMinimumWords(text, 1);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
_handleSpeechInterruption(active) {
|
|
316
|
+
if (this.snapshot.settings.mode !== 'headphones') return;
|
|
317
|
+
if (active) {
|
|
318
|
+
if (
|
|
319
|
+
!this.interruptionTranscriptConfirmed ||
|
|
320
|
+
this.interruptionTimer !== null ||
|
|
321
|
+
!this.snapshot.speaking ||
|
|
322
|
+
this.snapshot.paused
|
|
323
|
+
)
|
|
324
|
+
return;
|
|
325
|
+
this.interruptionTimer = setTimeout(() => {
|
|
326
|
+
this.interruptionTimer = null;
|
|
327
|
+
if (
|
|
328
|
+
this.interruptionTranscriptConfirmed &&
|
|
329
|
+
this.snapshot.recognizing &&
|
|
330
|
+
this.snapshot.speaking &&
|
|
331
|
+
!this.snapshot.paused
|
|
332
|
+
) {
|
|
333
|
+
this.interruptionPausedSpeech = true;
|
|
334
|
+
void this.pauseSpeech().then(() => {
|
|
335
|
+
if (
|
|
336
|
+
this.interruptionPausedSpeech &&
|
|
337
|
+
!this.snapshot.recognizing &&
|
|
338
|
+
this.snapshot.paused
|
|
339
|
+
) {
|
|
340
|
+
this.interruptionPausedSpeech = false;
|
|
341
|
+
void this.resumeSpeech();
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
}, this.snapshot.settings.assistantSpeechDelaySeconds * 1000);
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
if (this.interruptionTimer !== null) clearTimeout(this.interruptionTimer);
|
|
349
|
+
this.interruptionTimer = null;
|
|
350
|
+
this.interruptionTranscriptConfirmed = false;
|
|
351
|
+
if (this.interruptionPausedSpeech && this.snapshot.paused) {
|
|
352
|
+
this.interruptionPausedSpeech = false;
|
|
353
|
+
void this.resumeSpeech();
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
281
357
|
onResult({ final = '', interim = '' }) {
|
|
282
358
|
if (this.disposed || (!this.snapshot.listening && !this.snapshot.starting)) return;
|
|
283
|
-
if (
|
|
284
|
-
|
|
285
|
-
|
|
359
|
+
if (
|
|
360
|
+
this.snapshot.speaking &&
|
|
361
|
+
!this.snapshot.paused &&
|
|
362
|
+
this.snapshot.settings.mode === 'speaker'
|
|
363
|
+
)
|
|
364
|
+
return;
|
|
365
|
+
if (
|
|
366
|
+
this.snapshot.settings.mode === 'headphones' &&
|
|
367
|
+
this.snapshot.speaking &&
|
|
368
|
+
this._interruptionTranscriptQualifies(final || interim)
|
|
369
|
+
) {
|
|
370
|
+
this.interruptionTranscriptConfirmed = true;
|
|
371
|
+
this._handleSpeechInterruption(true);
|
|
372
|
+
}
|
|
373
|
+
// Pending structured questions own recognition while waiting for a hands-free
|
|
374
|
+
// answer. Keep their interim and final text out of the normal chat composer.
|
|
375
|
+
if (typeof this.composer.handleQuestionResult === 'function' &&
|
|
376
|
+
this.composer.handleQuestionResult({ final, interim })) {
|
|
377
|
+
this.cancelAutoSend();
|
|
378
|
+
this.transcript.reset();
|
|
379
|
+
this.patch({ recognizing: !!interim });
|
|
380
|
+
return;
|
|
286
381
|
}
|
|
287
382
|
// A native event can contain a final result without an interim result. Do not
|
|
288
383
|
// run a second empty hypothesis update: it would replace the just-committed
|
|
289
384
|
// result with a stale composer snapshot before the editor publishes it.
|
|
290
385
|
if (final) {
|
|
386
|
+
const command =
|
|
387
|
+
this.snapshot.settings.voiceCommandsEnabled &&
|
|
388
|
+
matchVoiceCommand(final, {
|
|
389
|
+
send: this.snapshot.settings.voiceCommandSend,
|
|
390
|
+
queue: this.snapshot.settings.voiceCommandQueue,
|
|
391
|
+
end: this.snapshot.settings.voiceCommandEnd,
|
|
392
|
+
mute: this.snapshot.settings.voiceCommandMute,
|
|
393
|
+
resume: this.snapshot.settings.voiceCommandResume,
|
|
394
|
+
stopSpeaking: this.snapshot.settings.voiceCommandStopSpeaking,
|
|
395
|
+
clear: this.snapshot.settings.voiceCommandClear,
|
|
396
|
+
});
|
|
397
|
+
if (command) {
|
|
398
|
+
this.composer.setDraft(this.transcript.update(this.composer.getDraft(), '', true));
|
|
399
|
+
this.transcript.reset();
|
|
400
|
+
this.cancelAutoSend();
|
|
401
|
+
this.patch({ recognizing: false });
|
|
402
|
+
if (command === 'end') void this.endConversation();
|
|
403
|
+
else if (command === 'mute') this.muteListening();
|
|
404
|
+
else if (command === 'resume') this.resumeListeningInput();
|
|
405
|
+
else if (command === 'stopSpeaking') {
|
|
406
|
+
this.updateSettings({ announceAssistantMessages: false });
|
|
407
|
+
void this.stopSpeech();
|
|
408
|
+
} else if (command === 'clear') {
|
|
409
|
+
this.cancelAutoSend();
|
|
410
|
+
this.transcript.reset();
|
|
411
|
+
this.composer.setDraft('');
|
|
412
|
+
} else if (this.snapshot.muted) return;
|
|
413
|
+
else {
|
|
414
|
+
const sendingMode = command === 'send' ? 'steer' : 'queue';
|
|
415
|
+
this.updateSettings({ sendingMode });
|
|
416
|
+
if (this.composer.getDraft().trim() && typeof this.composer.submit === 'function')
|
|
417
|
+
this.composer.submit(sendingMode);
|
|
418
|
+
}
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
if (this.snapshot.muted) {
|
|
422
|
+
this.composer.setDraft(this.transcript.update(this.composer.getDraft(), '', true));
|
|
423
|
+
this.transcript.reset();
|
|
424
|
+
this.patch({ recognizing: false });
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
if (
|
|
428
|
+
this.snapshot.settings.recognitionFilterEnabled &&
|
|
429
|
+
!hasMinimumWords(final, this.snapshot.settings.recognitionMinimumWords)
|
|
430
|
+
) {
|
|
431
|
+
this.composer.setDraft(this.transcript.update(this.composer.getDraft(), '', true));
|
|
432
|
+
this.transcript.reset();
|
|
433
|
+
this.patch({ recognizing: false });
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
291
436
|
const next = this.transcript.update(this.composer.getDraft(), final, true);
|
|
292
437
|
this.composer.setDraft(next);
|
|
293
|
-
if (this.snapshot.settings.sendingMode
|
|
438
|
+
if (this.snapshot.settings.sendingMode !== 'manual') this.scheduleAutoSend(next);
|
|
294
439
|
}
|
|
295
440
|
if (interim) {
|
|
296
441
|
this.cancelAutoSend();
|
|
@@ -315,10 +460,20 @@ export class VoiceCoordinator {
|
|
|
315
460
|
this.patch({ autoSendAt: null });
|
|
316
461
|
if (
|
|
317
462
|
!this.disposed &&
|
|
318
|
-
this.snapshot.settings.sendingMode
|
|
463
|
+
this.snapshot.settings.sendingMode !== 'manual' &&
|
|
319
464
|
expected === this.composer.getDraft()
|
|
320
|
-
)
|
|
321
|
-
|
|
465
|
+
) {
|
|
466
|
+
try {
|
|
467
|
+
this.composer.submit(this.snapshot.settings.sendingMode === 'steer' ? 'steer' : 'queue');
|
|
468
|
+
// Web Speech keeps a cumulative native result list for the lifetime of
|
|
469
|
+
// one recognition instance. Start a fresh instance at the turn boundary
|
|
470
|
+
// so the sent utterance cannot prefix the next one.
|
|
471
|
+
this.transcript.reset();
|
|
472
|
+
this.recognition.reset?.();
|
|
473
|
+
} catch (error) {
|
|
474
|
+
this.patch({ error: message(error) });
|
|
475
|
+
}
|
|
476
|
+
}
|
|
322
477
|
}, delay);
|
|
323
478
|
}
|
|
324
479
|
cancelAutoSend() {
|
|
@@ -356,6 +511,12 @@ export class VoiceCoordinator {
|
|
|
356
511
|
this.patch({ conversation: false });
|
|
357
512
|
await Promise.all([this.stopListening(), this.stopSpeech(false)]);
|
|
358
513
|
}
|
|
514
|
+
_syncSpeechSegments() {
|
|
515
|
+
const speechSegmentsRemaining = this.queue.length + (this.snapshot.speaking ? 1 : 0);
|
|
516
|
+
if (speechSegmentsRemaining !== this.snapshot.speechSegmentsRemaining)
|
|
517
|
+
this.patch({ speechSegmentsRemaining });
|
|
518
|
+
}
|
|
519
|
+
|
|
359
520
|
_suppressPending() {
|
|
360
521
|
for (const id of this.unfinished) this.suppressed.add(id);
|
|
361
522
|
if (this.snapshot.activeMessageId !== null) this.suppressed.add(this.snapshot.activeMessageId);
|
|
@@ -363,10 +524,15 @@ export class VoiceCoordinator {
|
|
|
363
524
|
this.queue = [];
|
|
364
525
|
}
|
|
365
526
|
async stopSpeech(resumeListening = true) {
|
|
527
|
+
if (this.interruptionTimer !== null) clearTimeout(this.interruptionTimer);
|
|
528
|
+
this.interruptionTimer = null;
|
|
529
|
+
this.interruptionTranscriptConfirmed = false;
|
|
530
|
+
this.interruptionPausedSpeech = false;
|
|
366
531
|
const epoch = ++this.speechEpoch;
|
|
367
532
|
++this.controlEpoch;
|
|
368
533
|
this._suppressPending();
|
|
369
534
|
this.patch({ speaking: false, paused: false, activeMessageId: null });
|
|
535
|
+
this._syncSpeechSegments();
|
|
370
536
|
const stopped = this.speechBarrier.then(async () => {
|
|
371
537
|
const results = await Promise.allSettled(
|
|
372
538
|
Object.values(this.engines).map((engine) => Promise.resolve().then(() => engine.stop())),
|
|
@@ -431,13 +597,23 @@ export class VoiceCoordinator {
|
|
|
431
597
|
const stopped = this.stopSpeech(false);
|
|
432
598
|
const epoch = this.speechEpoch;
|
|
433
599
|
await stopped;
|
|
434
|
-
if (
|
|
435
|
-
|
|
600
|
+
if (this.disposed || epoch !== this.speechEpoch || this.speechStopError) return;
|
|
601
|
+
const segments = splitSpeechOutput(text, {
|
|
602
|
+
filterCodeBlocks: this.snapshot.settings.outputCodeFilterEnabled,
|
|
603
|
+
codeBlockMaxLines: this.snapshot.settings.outputCodeMaxLines,
|
|
604
|
+
codeBlockNotice: this.snapshot.settings.outputCodeNotice,
|
|
605
|
+
});
|
|
606
|
+
const first = segments.shift();
|
|
607
|
+
if (!first) return;
|
|
608
|
+
this.queue.push(...segments.map((segment) => ({ text: segment, id: messageId, manual: true })));
|
|
609
|
+
this._syncSpeechSegments();
|
|
610
|
+
return this.play(first, messageId, true);
|
|
436
611
|
}
|
|
437
|
-
async play(text, messageId) {
|
|
612
|
+
async play(text, messageId, manual = false) {
|
|
438
613
|
if (this.disposed || !text.trim()) return;
|
|
439
614
|
if (this.snapshot.speaking) {
|
|
440
|
-
this.queue.push({ text, id: messageId });
|
|
615
|
+
this.queue.push({ text, id: messageId, manual });
|
|
616
|
+
this._syncSpeechSegments();
|
|
441
617
|
return;
|
|
442
618
|
}
|
|
443
619
|
const epoch = ++this.speechEpoch;
|
|
@@ -449,6 +625,7 @@ export class VoiceCoordinator {
|
|
|
449
625
|
}
|
|
450
626
|
// Reserve playback synchronously so incoming stream chunks cannot overtake gating.
|
|
451
627
|
this.patch({ speaking: true, paused: false, activeMessageId: messageId, error: null });
|
|
628
|
+
this._syncSpeechSegments();
|
|
452
629
|
let failed = false;
|
|
453
630
|
try {
|
|
454
631
|
await this.speechBarrier;
|
|
@@ -462,6 +639,7 @@ export class VoiceCoordinator {
|
|
|
462
639
|
await engine.speak(text, {
|
|
463
640
|
voice: this.snapshot.settings.voice || undefined,
|
|
464
641
|
rate: this.snapshot.settings.rate,
|
|
642
|
+
outputDeviceId: this.snapshot.settings.outputDeviceId,
|
|
465
643
|
});
|
|
466
644
|
} catch (error) {
|
|
467
645
|
if (epoch === this.speechEpoch) {
|
|
@@ -472,6 +650,7 @@ export class VoiceCoordinator {
|
|
|
472
650
|
} finally {
|
|
473
651
|
if (epoch === this.speechEpoch && !this.disposed) {
|
|
474
652
|
this.patch({ speaking: false, paused: false, activeMessageId: null });
|
|
653
|
+
this._syncSpeechSegments();
|
|
475
654
|
if (!failed && this.queue.length) this._drain();
|
|
476
655
|
else if (this.snapshot.conversation && !this.snapshot.listening && !this.snapshot.starting)
|
|
477
656
|
await this._startInput(true);
|
|
@@ -485,8 +664,8 @@ export class VoiceCoordinator {
|
|
|
485
664
|
_drain() {
|
|
486
665
|
if (
|
|
487
666
|
this.disposed ||
|
|
488
|
-
!this.snapshot.conversation ||
|
|
489
|
-
!this.snapshot.settings.announceAssistantMessages ||
|
|
667
|
+
(!this.snapshot.conversation && !this.queue[0]?.manual) ||
|
|
668
|
+
(!this.snapshot.settings.announceAssistantMessages && !this.queue[0]?.manual) ||
|
|
490
669
|
this.snapshot.speaking ||
|
|
491
670
|
this.snapshot.recognizing ||
|
|
492
671
|
this.snapshot.starting ||
|
|
@@ -505,7 +684,7 @@ export class VoiceCoordinator {
|
|
|
505
684
|
this._cancelAssistantSpeechTimer();
|
|
506
685
|
this.assistantSpeechNotBefore = 0;
|
|
507
686
|
const next = this.queue.shift();
|
|
508
|
-
void this.play(next.text, next.id);
|
|
687
|
+
void this.play(next.text, next.id, next.manual === true);
|
|
509
688
|
}
|
|
510
689
|
/** Baselines survive conversation toggles; cancelled message IDs remain suppressed. */
|
|
511
690
|
observeMessage(id, text, { complete = false, baseline = false } = {}) {
|
|
@@ -527,19 +706,23 @@ export class VoiceCoordinator {
|
|
|
527
706
|
return;
|
|
528
707
|
}
|
|
529
708
|
const remaining = text.slice(offset);
|
|
530
|
-
const boundary = complete
|
|
531
|
-
? remaining.length
|
|
532
|
-
: Math.max(
|
|
533
|
-
remaining.lastIndexOf('. '),
|
|
534
|
-
remaining.lastIndexOf('? '),
|
|
535
|
-
remaining.lastIndexOf('! '),
|
|
536
|
-
remaining.lastIndexOf('\n'),
|
|
537
|
-
) + 1;
|
|
709
|
+
const boundary = complete ? remaining.length : remaining.lastIndexOf('\n') + 1;
|
|
538
710
|
if (boundary <= 0) return;
|
|
539
|
-
|
|
711
|
+
if (
|
|
712
|
+
this.snapshot.settings.outputCodeFilterEnabled &&
|
|
713
|
+
hasUnclosedCodeFence(remaining.slice(0, boundary)) &&
|
|
714
|
+
!complete
|
|
715
|
+
)
|
|
716
|
+
return;
|
|
717
|
+
const chunk = filterSpeechOutput(remaining.slice(0, boundary).trim(), {
|
|
718
|
+
filterCodeBlocks: this.snapshot.settings.outputCodeFilterEnabled,
|
|
719
|
+
codeBlockMaxLines: this.snapshot.settings.outputCodeMaxLines,
|
|
720
|
+
codeBlockNotice: this.snapshot.settings.outputCodeNotice,
|
|
721
|
+
}).trim();
|
|
540
722
|
this.consumed.set(id, offset + boundary);
|
|
541
723
|
if (chunk) {
|
|
542
724
|
this.queue.push({ text: chunk, id });
|
|
725
|
+
this._syncSpeechSegments();
|
|
543
726
|
this._drain();
|
|
544
727
|
}
|
|
545
728
|
}
|
|
@@ -549,6 +732,6 @@ export class VoiceCoordinator {
|
|
|
549
732
|
this._cancelAssistantSpeechTimer();
|
|
550
733
|
this.disposed = true;
|
|
551
734
|
this.listeners.clear();
|
|
552
|
-
await this.endConversation();
|
|
735
|
+
await this.endConversation(true);
|
|
553
736
|
}
|
|
554
737
|
}
|