dsh-live-voice 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -14
- package/lib/client.js +6474 -3396
- package/lib/server.js +29 -6
- package/package.json +9 -30
- package/src/app/AGENTS.md +1 -1
- package/src/app/ARCHITECTURE.md +1 -1
- package/src/app/client/apply.tsx +242 -35
- package/src/app/client/composerSelection.ts +92 -0
- package/src/app/client/i18n/catalogs/base.ts +35 -0
- package/src/app/client/i18n/catalogs/en.ts +43 -3
- package/src/app/client/i18n/catalogs/es.ts +51 -11
- package/src/app/client/i18n/catalogs/fr.ts +52 -11
- package/src/app/client/i18n/catalogs/hi.ts +50 -11
- package/src/app/client/i18n/catalogs/pt-BR.ts +52 -11
- package/src/app/client/i18n/catalogs/zh.ts +50 -11
- package/src/modules/AGENTS.md +1 -1
- package/src/modules/conversation/ARCHITECTURE.md +6 -2
- package/src/modules/conversation/components/AutoPlaybackToggle.tsx +31 -0
- package/src/modules/conversation/components/ConversationControls.tsx +23 -8
- package/src/modules/conversation/components/ConversationStatusBar.tsx +91 -93
- package/src/modules/conversation/components/DeliveryModeButton.tsx +3 -3
- package/src/modules/conversation/components/MeetingControls.tsx +40 -0
- package/src/modules/conversation/components/MicrophoneButton.tsx +15 -2
- package/src/modules/conversation/components/PlaybackControls.tsx +20 -5
- package/src/modules/conversation/components/RecognitionBar.tsx +14 -0
- package/src/modules/conversation/components/ScrollingSpeechCaption.tsx +100 -0
- package/src/modules/conversation/components/SpeechStatusBar.tsx +99 -0
- package/src/modules/conversation/components/Waveform.tsx +8 -3
- package/src/modules/conversation/components/conversationStatus.ts +1 -3
- package/src/modules/conversation/components/createConversationComponents.tsx +1 -1
- package/src/modules/conversation/components/index.ts +1 -0
- package/src/modules/conversation/components/speechCaption.ts +27 -0
- package/src/modules/conversation/models/chat.ts +15 -2
- package/src/modules/conversation/models/meeting.ts +256 -0
- package/src/modules/conversation/models/meetingTranscript.ts +63 -0
- package/src/modules/core/ARCHITECTURE.md +1 -1
- package/src/modules/core/coordinator.ts +353 -81
- package/src/modules/core/developerExtension.ts +26 -0
- package/src/modules/core/diagnostics.ts +153 -0
- package/src/modules/core/filters.ts +20 -15
- package/src/modules/core/markdownSpeech.ts +154 -0
- package/src/modules/core/settings.ts +17 -13
- package/src/modules/core/sharedAudio.ts +88 -0
- package/src/modules/core/speechDefaults.ts +21 -0
- package/src/modules/recognition/engines/browser/BrowserRecognitionEngine.ts +8 -1
- package/src/modules/recognition/engines/whisper/WhisperRecognitionEngine.ts +10 -3
- package/src/modules/settings/components/LiveVoiceSettings.tsx +17 -3
- package/src/modules/settings/components/SettingsHeader.tsx +1 -1
- package/src/modules/settings/models/settingsStorage.ts +11 -3
- package/src/modules/settings/sections/conversation/ConversationDelaySettings.tsx +8 -3
- package/src/modules/settings/sections/conversation/DeliverySettings.tsx +25 -2
- package/src/modules/settings/sections/recognition/RecognitionEngineSettings.tsx +2 -1
- package/src/modules/settings/sections/speak/SpeechEngineSettings.tsx +12 -1
- package/src/modules/speak/engines/audio/HostAudioEngine.ts +19 -2
- package/src/modules/speak/engines/browser/BrowserSpeakingEngine.ts +33 -2
- package/src/shared/design-system/buttons/ToggleButton.tsx +8 -1
- package/src/shared/design-system/icons/icons.ts +6 -0
- package/src/styles/index.ts +13 -3
- package/CHANGELOG.md +0 -240
- package/DEVELOPMENT.md +0 -83
- package/PLAN.md +0 -296
- package/docs/ARCHITECTURE.md +0 -33
- package/docs/CHOOSING-AN-ENGINE.md +0 -120
- package/docs/CONFIGURATION.md +0 -163
- package/docs/REVIEW.md +0 -35
- package/docs/VOICE-LIFECYCLE.md +0 -63
- package/scripts/build.ts +0 -62
- package/scripts/check-dist.mjs +0 -38
- package/scripts/preview-ui.ts +0 -31
- package/scripts/probe-browser.ts +0 -46
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import { Waveform } from './Waveform.js';
|
|
3
|
+
export function RecognitionBar({ controller, listening, label, status, leading, children }: any) {
|
|
4
|
+
return (
|
|
5
|
+
<div className="dlv-pill" role="group" aria-label={label}>
|
|
6
|
+
{leading}
|
|
7
|
+
<Waveform controller={controller} enabled={Boolean(listening)} />
|
|
8
|
+
<span className="dlv-status" role="status" aria-live="polite">
|
|
9
|
+
{status}
|
|
10
|
+
</span>
|
|
11
|
+
{children}
|
|
12
|
+
</div>
|
|
13
|
+
);
|
|
14
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
|
|
3
|
+
/** Visual approximation only: no word-level timing is available. */
|
|
4
|
+
export function ScrollingSpeechCaption({
|
|
5
|
+
text,
|
|
6
|
+
label,
|
|
7
|
+
controller,
|
|
8
|
+
segment,
|
|
9
|
+
paused,
|
|
10
|
+
loading,
|
|
11
|
+
}: {
|
|
12
|
+
text: string;
|
|
13
|
+
label: string;
|
|
14
|
+
controller: any;
|
|
15
|
+
segment: number;
|
|
16
|
+
paused: boolean;
|
|
17
|
+
loading: boolean;
|
|
18
|
+
}) {
|
|
19
|
+
const viewport = React.useRef<HTMLDivElement>(null);
|
|
20
|
+
const line = React.useRef<HTMLSpanElement>(null);
|
|
21
|
+
const marker = React.useRef<HTMLSpanElement>(null);
|
|
22
|
+
const current = React.useRef({ paused, loading });
|
|
23
|
+
current.current = { paused, loading };
|
|
24
|
+
React.useEffect(() => {
|
|
25
|
+
const box = viewport.current;
|
|
26
|
+
const content = line.current;
|
|
27
|
+
const band = marker.current;
|
|
28
|
+
if (!box || !content || !band) return;
|
|
29
|
+
const host = box.ownerDocument.defaultView!;
|
|
30
|
+
let width = box.clientWidth;
|
|
31
|
+
let textWidth = content.scrollWidth;
|
|
32
|
+
let fraction = 0;
|
|
33
|
+
let previousTime = 0;
|
|
34
|
+
let frame = 0;
|
|
35
|
+
box.scrollLeft = 0;
|
|
36
|
+
band.style.transform = 'translateX(0px)';
|
|
37
|
+
const measure = () => {
|
|
38
|
+
width = box.clientWidth;
|
|
39
|
+
textWidth = content.scrollWidth;
|
|
40
|
+
};
|
|
41
|
+
const observer =
|
|
42
|
+
typeof host.ResizeObserver === 'function' ? new host.ResizeObserver(measure) : null;
|
|
43
|
+
observer?.observe(box);
|
|
44
|
+
observer?.observe(content);
|
|
45
|
+
const reduced = host.matchMedia?.('(prefers-reduced-motion: reduce)').matches;
|
|
46
|
+
const tick = (time: number) => {
|
|
47
|
+
const delta = Math.min(100, previousTime ? time - previousTime : 16);
|
|
48
|
+
previousTime = time;
|
|
49
|
+
const state = current.current;
|
|
50
|
+
if (!state.paused && !state.loading) {
|
|
51
|
+
const progress = controller.getSpeechProgress?.();
|
|
52
|
+
if (
|
|
53
|
+
progress &&
|
|
54
|
+
Number.isFinite(progress.durationSeconds) &&
|
|
55
|
+
progress.durationSeconds > 0 &&
|
|
56
|
+
Number.isFinite(progress.positionSeconds)
|
|
57
|
+
) {
|
|
58
|
+
const target = Math.max(
|
|
59
|
+
0,
|
|
60
|
+
Math.min(1, progress.positionSeconds / progress.durationSeconds),
|
|
61
|
+
);
|
|
62
|
+
fraction = reduced
|
|
63
|
+
? target
|
|
64
|
+
: fraction + (target - fraction) * (1 - Math.exp(-delta / 90));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const bandWidth = Math.min(90, width, textWidth);
|
|
68
|
+
const position = fraction * Math.max(0, textWidth - bandWidth);
|
|
69
|
+
const scroll = Math.max(
|
|
70
|
+
0,
|
|
71
|
+
Math.min(Math.max(0, textWidth - width), position - (width - bandWidth) / 2),
|
|
72
|
+
);
|
|
73
|
+
box.scrollLeft = scroll;
|
|
74
|
+
band.style.width = bandWidth + 'px';
|
|
75
|
+
band.style.transform = 'translateX(' + Math.max(0, position - box.scrollLeft) + 'px)';
|
|
76
|
+
band.style.opacity = state.loading ? '0' : '1';
|
|
77
|
+
frame = host.requestAnimationFrame(tick);
|
|
78
|
+
};
|
|
79
|
+
frame = host.requestAnimationFrame(tick);
|
|
80
|
+
return () => {
|
|
81
|
+
host.cancelAnimationFrame(frame);
|
|
82
|
+
observer?.disconnect();
|
|
83
|
+
};
|
|
84
|
+
}, [controller, text, segment]);
|
|
85
|
+
return (
|
|
86
|
+
<div className="dlv-caption-stage">
|
|
87
|
+
<div
|
|
88
|
+
ref={viewport}
|
|
89
|
+
className="dlv-caption dlv-caption-scroll"
|
|
90
|
+
title={text}
|
|
91
|
+
aria-label={label}
|
|
92
|
+
>
|
|
93
|
+
<span ref={line} className="dlv-caption-line">
|
|
94
|
+
{text}
|
|
95
|
+
</span>
|
|
96
|
+
</div>
|
|
97
|
+
<span ref={marker} className="dlv-caption-marker" aria-hidden="true" />
|
|
98
|
+
</div>
|
|
99
|
+
);
|
|
100
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import { useLanguage } from '../../../app/client/i18n/index.js';
|
|
3
|
+
import { ErrorMessage, IconButton } from '../../../shared/design-system/index.js';
|
|
4
|
+
import { useConversationActions, useConversationController } from '../hooks/index.js';
|
|
5
|
+
import { PlaybackControls } from './PlaybackControls.js';
|
|
6
|
+
import { ScrollingSpeechCaption } from './ScrollingSpeechCaption.js';
|
|
7
|
+
export function SpeechStatusBar({ controller }: { controller: any }) {
|
|
8
|
+
const { scoped: speak } = useLanguage((ctx) => ctx.speak);
|
|
9
|
+
const { scoped: commons } = useLanguage((ctx) => ctx.commons);
|
|
10
|
+
const state = useConversationController<any>(controller);
|
|
11
|
+
const { invoke, error, clearError } = useConversationActions(controller);
|
|
12
|
+
const [progress, setProgress] = React.useState<any>(null);
|
|
13
|
+
React.useEffect(() => {
|
|
14
|
+
setProgress(null);
|
|
15
|
+
if (!state.speechText || !state.speaking) return;
|
|
16
|
+
const update = () => setProgress(controller.getSpeechProgress?.() ?? null);
|
|
17
|
+
update();
|
|
18
|
+
if (state.paused) return;
|
|
19
|
+
const timer = setInterval(update, 100);
|
|
20
|
+
return () => clearInterval(timer);
|
|
21
|
+
}, [controller, state.speechText, state.speaking, state.paused]);
|
|
22
|
+
if (
|
|
23
|
+
!state.speaking &&
|
|
24
|
+
!state.paused &&
|
|
25
|
+
!state.speechRunActive &&
|
|
26
|
+
!(state.speechSegmentsRemaining > 0)
|
|
27
|
+
)
|
|
28
|
+
return null;
|
|
29
|
+
const text = state.speechText ?? '';
|
|
30
|
+
const ratio =
|
|
31
|
+
progress &&
|
|
32
|
+
Number.isFinite(progress.durationSeconds) &&
|
|
33
|
+
progress.durationSeconds > 0 &&
|
|
34
|
+
Number.isFinite(progress.positionSeconds)
|
|
35
|
+
? Math.max(0, Math.min(1, progress.positionSeconds / progress.durationSeconds))
|
|
36
|
+
: null;
|
|
37
|
+
const loading = state.speechLoading === true && state.speaking && !state.paused;
|
|
38
|
+
const index = state.speechSegmentIndex ?? 0;
|
|
39
|
+
const total = state.speechSegmentsTotal ?? state.speechSegmentsRemaining ?? 0;
|
|
40
|
+
return (
|
|
41
|
+
<div className="dlv-bar-wrap dlv-speech-bar">
|
|
42
|
+
<div
|
|
43
|
+
className="dlv-pill dlv-speech-pill"
|
|
44
|
+
role="group"
|
|
45
|
+
aria-label={(speak as any).captions.title()}
|
|
46
|
+
>
|
|
47
|
+
<IconButton
|
|
48
|
+
label={(speak as any).playback.previous()}
|
|
49
|
+
icon="skipPrevious"
|
|
50
|
+
disabled={!state.speechHasPrevious}
|
|
51
|
+
onClick={() => invoke('previousSpeechSegment')}
|
|
52
|
+
/>
|
|
53
|
+
<div className="dlv-caption-stack">
|
|
54
|
+
<ScrollingSpeechCaption
|
|
55
|
+
text={text || (speak as any).status.playing()}
|
|
56
|
+
label={(speak as any).captions.approximate()}
|
|
57
|
+
controller={controller}
|
|
58
|
+
segment={index}
|
|
59
|
+
paused={state.paused || !state.speaking}
|
|
60
|
+
loading={loading}
|
|
61
|
+
/>
|
|
62
|
+
<div
|
|
63
|
+
className="dlv-caption-progress"
|
|
64
|
+
data-loading={loading ? 'true' : 'false'}
|
|
65
|
+
role="progressbar"
|
|
66
|
+
aria-hidden={!loading}
|
|
67
|
+
aria-label={
|
|
68
|
+
loading ? (speak as any).captions.loading() : (speak as any).captions.progress()
|
|
69
|
+
}
|
|
70
|
+
aria-valuemin={0}
|
|
71
|
+
aria-valuemax={100}
|
|
72
|
+
aria-valuenow={loading || ratio === null ? undefined : Math.round(ratio * 100)}
|
|
73
|
+
>
|
|
74
|
+
<span style={loading ? undefined : { width: (ratio ?? 0) * 100 + '%' }} />
|
|
75
|
+
</div>
|
|
76
|
+
</div>
|
|
77
|
+
<IconButton
|
|
78
|
+
label={(speak as any).playback.next()}
|
|
79
|
+
icon="skipNext"
|
|
80
|
+
disabled={!(state.speechSegmentsRemaining > 1)}
|
|
81
|
+
onClick={() => invoke('skipSpeechSegment')}
|
|
82
|
+
/>
|
|
83
|
+
<span
|
|
84
|
+
className="dlv-speech-count"
|
|
85
|
+
aria-label={(speak as any).captions.position({ index, total })}
|
|
86
|
+
>
|
|
87
|
+
{index}/{total}
|
|
88
|
+
</span>
|
|
89
|
+
<PlaybackControls state={state} invoke={invoke} navigation={false} />
|
|
90
|
+
</div>
|
|
91
|
+
<ErrorMessage
|
|
92
|
+
error={error}
|
|
93
|
+
dismissLabel={(commons as any).dismissError()}
|
|
94
|
+
dismissText={(commons as any).dismiss()}
|
|
95
|
+
onDismiss={clearError}
|
|
96
|
+
/>
|
|
97
|
+
</div>
|
|
98
|
+
);
|
|
99
|
+
}
|
|
@@ -3,6 +3,8 @@ import React from 'react';
|
|
|
3
3
|
export type WaveformProps = { controller: any; enabled: boolean };
|
|
4
4
|
export function Waveform({ controller, enabled }: WaveformProps) {
|
|
5
5
|
const ref = React.useRef<HTMLCanvasElement>(null);
|
|
6
|
+
const current = React.useRef({ controller, enabled });
|
|
7
|
+
current.current = { controller, enabled };
|
|
6
8
|
React.useEffect(() => {
|
|
7
9
|
const canvas = ref.current;
|
|
8
10
|
const context = canvas?.getContext('2d');
|
|
@@ -18,8 +20,10 @@ export function Waveform({ controller, enabled }: WaveformProps) {
|
|
|
18
20
|
width = Math.max(1, bounds.width);
|
|
19
21
|
height = Math.max(1, bounds.height || 40);
|
|
20
22
|
ratio = Math.max(1, window.devicePixelRatio || 1);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
const nextWidth = Math.round(width * ratio);
|
|
24
|
+
const nextHeight = Math.round(height * ratio);
|
|
25
|
+
if (canvas.width !== nextWidth) canvas.width = nextWidth;
|
|
26
|
+
if (canvas.height !== nextHeight) canvas.height = nextHeight;
|
|
23
27
|
}
|
|
24
28
|
const observer = typeof ResizeObserver === 'function' ? new ResizeObserver(resize) : null;
|
|
25
29
|
observer?.observe(canvas);
|
|
@@ -30,6 +34,7 @@ export function Waveform({ controller, enabled }: WaveformProps) {
|
|
|
30
34
|
if (ratio !== Math.max(1, window.devicePixelRatio || 1)) resize();
|
|
31
35
|
context.setTransform(ratio, 0, 0, ratio, 0, 0);
|
|
32
36
|
context.clearRect(0, 0, width, height);
|
|
37
|
+
const { controller, enabled } = current.current;
|
|
33
38
|
const raw = Number(controller.meter?.level?.() ?? 0);
|
|
34
39
|
const level = enabled && Number.isFinite(raw) ? Math.min(1, Math.max(0, raw)) : 0;
|
|
35
40
|
const color = getComputedStyle(canvas).color;
|
|
@@ -63,6 +68,6 @@ export function Waveform({ controller, enabled }: WaveformProps) {
|
|
|
63
68
|
observer?.disconnect();
|
|
64
69
|
window.removeEventListener('resize', resize);
|
|
65
70
|
};
|
|
66
|
-
}, [
|
|
71
|
+
}, []);
|
|
67
72
|
return <canvas ref={ref} className="dlv-wave" aria-hidden />;
|
|
68
73
|
}
|
|
@@ -8,9 +8,7 @@ export function resolveConversationStatus(
|
|
|
8
8
|
if (state.answeringQuestion && state.listening) return recognition.status.awaitingAnswer();
|
|
9
9
|
if (remaining) return settings.autoSend.countdown({ remaining });
|
|
10
10
|
if (state.starting) return recognition.microphone.starting();
|
|
11
|
-
if (state.
|
|
12
|
-
if (state.speaking) return speak.status.playing();
|
|
13
|
-
if (state.recognizing) return recognition.status.processing();
|
|
11
|
+
if (state.recognizing || state.pendingTranscriptions > 0) return recognition.status.processing();
|
|
14
12
|
if (state.listening) return recognition.status.listening();
|
|
15
13
|
if (state.conversation) return commons.conversation.idle();
|
|
16
14
|
return commons.status.ready();
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Approximate character timing; keep the current word visible. */
|
|
2
|
+
export function remainingSpeechCaption(
|
|
3
|
+
text: string,
|
|
4
|
+
progress: { positionSeconds: number; durationSeconds: number } | null,
|
|
5
|
+
) {
|
|
6
|
+
if (
|
|
7
|
+
!progress ||
|
|
8
|
+
!Number.isFinite(progress.durationSeconds) ||
|
|
9
|
+
progress.durationSeconds <= 0 ||
|
|
10
|
+
!Number.isFinite(progress.positionSeconds)
|
|
11
|
+
)
|
|
12
|
+
return text;
|
|
13
|
+
const characters = Array.from(text);
|
|
14
|
+
let index = Math.min(
|
|
15
|
+
characters.length - 1,
|
|
16
|
+
Math.floor(
|
|
17
|
+
characters.length *
|
|
18
|
+
Math.max(0, Math.min(1, progress.positionSeconds / progress.durationSeconds)),
|
|
19
|
+
),
|
|
20
|
+
);
|
|
21
|
+
if (/\s/u.test(characters[index])) {
|
|
22
|
+
while (index < characters.length - 1 && /\s/u.test(characters[index])) index++;
|
|
23
|
+
} else if (/\s/u.test(text)) {
|
|
24
|
+
while (index > 0 && !/\s/u.test(characters[index - 1])) index--;
|
|
25
|
+
}
|
|
26
|
+
return characters.slice(Math.max(0, index)).join('').trimStart();
|
|
27
|
+
}
|
|
@@ -15,6 +15,7 @@ export function assistantMessages(snapshot) {
|
|
|
15
15
|
messageId: data.finalNode?.messageId,
|
|
16
16
|
turn: data.turn,
|
|
17
17
|
step: data.step,
|
|
18
|
+
time: data.time,
|
|
18
19
|
complete: data.status !== 'running',
|
|
19
20
|
interrupted: data.status === 'interrupted',
|
|
20
21
|
text: (data.blocks || [])
|
|
@@ -42,7 +43,19 @@ export function latestUserSequence(snapshot) {
|
|
|
42
43
|
);
|
|
43
44
|
}
|
|
44
45
|
export function addressedTurn(messages, messageId) {
|
|
45
|
-
|
|
46
|
+
if (messageId == null) return { text: '', ids: [] };
|
|
47
|
+
const addressed = messages.find(
|
|
48
|
+
(message) => message.messageId != null && String(message.messageId) === String(messageId),
|
|
49
|
+
);
|
|
46
50
|
if (!addressed) return { text: '', ids: [] };
|
|
47
|
-
|
|
51
|
+
const turn = messages
|
|
52
|
+
.filter((message) => message.turn === addressed.turn)
|
|
53
|
+
.sort((a, b) => a.step - b.step);
|
|
54
|
+
return {
|
|
55
|
+
text: turn
|
|
56
|
+
.map((message) => message.text)
|
|
57
|
+
.filter((text) => text.trim())
|
|
58
|
+
.join('\n\n'),
|
|
59
|
+
id: addressed.id,
|
|
60
|
+
};
|
|
48
61
|
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
import { MeetingTranscript } from './meetingTranscript.js';
|
|
3
|
+
import { VoiceCoordinator } from '../../core/coordinator.ts';
|
|
4
|
+
|
|
5
|
+
/** Application-owned capture; each source uses the same independent input policy. */
|
|
6
|
+
export class MeetingController {
|
|
7
|
+
constructor({ composer, settings, createSource, translate, onActivity, onSpeech, stopSpeech }) {
|
|
8
|
+
this.composer = composer;
|
|
9
|
+
this.stopSpeech = stopSpeech;
|
|
10
|
+
this.inputs = new Map();
|
|
11
|
+
this.inputOverrides = new Map();
|
|
12
|
+
this.settings = settings;
|
|
13
|
+
this.createSource = createSource;
|
|
14
|
+
this.t = translate;
|
|
15
|
+
this.onActivity = onActivity;
|
|
16
|
+
this.onSpeech = onSpeech;
|
|
17
|
+
this.transcript = new MeetingTranscript(composer);
|
|
18
|
+
this.jobs = new Map();
|
|
19
|
+
this.generations = new Map();
|
|
20
|
+
this.listeners = new Set();
|
|
21
|
+
this.snapshot = {
|
|
22
|
+
active: false,
|
|
23
|
+
timestamps: false,
|
|
24
|
+
microphone: { starting: false, listening: false, pending: 0, error: null },
|
|
25
|
+
shared: { starting: false, listening: false, pending: 0, error: null },
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
getSnapshot = () => this.snapshot;
|
|
29
|
+
subscribe = (listener) => {
|
|
30
|
+
this.listeners.add(listener);
|
|
31
|
+
return () => this.listeners.delete(listener);
|
|
32
|
+
};
|
|
33
|
+
patch(source, next) {
|
|
34
|
+
if (source) {
|
|
35
|
+
const input = this.inputs.get(source) || (next.listening || next.starting || next.error ? this.input(source) : null);
|
|
36
|
+
const policy = {};
|
|
37
|
+
for (const key of ['starting', 'listening', 'error'])
|
|
38
|
+
if (Object.hasOwn(next, key)) policy[key] = next[key];
|
|
39
|
+
if (Object.hasOwn(next, 'pending')) policy.pendingTranscriptions = next.pending;
|
|
40
|
+
input?.patch(policy);
|
|
41
|
+
}
|
|
42
|
+
this.snapshot = {
|
|
43
|
+
...this.snapshot,
|
|
44
|
+
...(source ? { [source]: { ...this.snapshot[source], ...next } } : next),
|
|
45
|
+
};
|
|
46
|
+
for (const listener of this.listeners) listener();
|
|
47
|
+
}
|
|
48
|
+
input(source = 'shared') {
|
|
49
|
+
if (this.inputs.has(source)) return this.inputs.get(source);
|
|
50
|
+
// Capture remains application-owned. This coordinator owns only source policy;
|
|
51
|
+
// it never acquires the microphone or queues a second assistant playback.
|
|
52
|
+
const input = new VoiceCoordinator({
|
|
53
|
+
recognition: { stop: async () => {} }, engines: {},
|
|
54
|
+
meter: { stop: async () => {} }, settings: { ...this.settings(), microphoneEnabled: true },
|
|
55
|
+
composer: {
|
|
56
|
+
getDraft: () => this.composer.getDraft(),
|
|
57
|
+
setDraft: (text) => this.composer.setDraft(text),
|
|
58
|
+
appendFinal: (text, startedAt) => {
|
|
59
|
+
this.transcript.append(source, text, startedAt);
|
|
60
|
+
return this.composer.getDraft();
|
|
61
|
+
},
|
|
62
|
+
submit: (mode) => this.composer.submit?.(mode),
|
|
63
|
+
handleQuestionResult: (result) => this.composer.handleQuestionResult?.(result),
|
|
64
|
+
canAutoSend: () => this.composer.canAutoSend?.() !== false,
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
this.inputOverrides.set(source, {
|
|
68
|
+
sendingMode: input.getSnapshot().settings.sendingMode,
|
|
69
|
+
voiceCommandsEnabled: input.getSnapshot().settings.voiceCommandsEnabled,
|
|
70
|
+
});
|
|
71
|
+
const update = input.updateSettings.bind(input);
|
|
72
|
+
input.updateSettings = (next) => {
|
|
73
|
+
this.inputOverrides.set(source, { ...this.inputOverrides.get(source), ...next });
|
|
74
|
+
update(next);
|
|
75
|
+
};
|
|
76
|
+
input.clearError = () => this.patch(source, { error: null });
|
|
77
|
+
input.endConversation = () => this.stop(source);
|
|
78
|
+
input.stopSpeech = () => this.stopSpeech?.();
|
|
79
|
+
const updatePolicy = input.updateSettings;
|
|
80
|
+
input.updateSettings = (next) => {
|
|
81
|
+
if (Object.hasOwn(next, 'announceAssistantMessages'))
|
|
82
|
+
this.composer.updatePlaybackSettings?.({ announceAssistantMessages: next.announceAssistantMessages });
|
|
83
|
+
return updatePolicy(next);
|
|
84
|
+
};
|
|
85
|
+
input.toggleTimestamps = () => {
|
|
86
|
+
const timestamps = !input.getSnapshot().timestamps;
|
|
87
|
+
this.transcript.setSourceTimestamps(source, timestamps);
|
|
88
|
+
input.patch({ timestamps });
|
|
89
|
+
};
|
|
90
|
+
this.inputs.set(source, input);
|
|
91
|
+
return input;
|
|
92
|
+
}
|
|
93
|
+
applySettings(settings) {
|
|
94
|
+
for (const [source, input] of this.inputs)
|
|
95
|
+
VoiceCoordinator.prototype.updateSettings.call(input, {
|
|
96
|
+
...settings, ...this.inputOverrides.get(source),
|
|
97
|
+
microphoneEnabled: !input.getSnapshot().muted,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
composerChanged(draft) {
|
|
101
|
+
for (const input of this.inputs.values()) input.composerChanged(draft);
|
|
102
|
+
}
|
|
103
|
+
refreshDeliveryReadiness() {
|
|
104
|
+
for (const input of this.inputs.values()) input.refreshDeliveryReadiness();
|
|
105
|
+
}
|
|
106
|
+
cancelDelivery() {
|
|
107
|
+
for (const input of this.inputs.values()) input.cancelAutoSend();
|
|
108
|
+
}
|
|
109
|
+
toggleTimestamps() {
|
|
110
|
+
const enabled = !this.snapshot.timestamps;
|
|
111
|
+
this.transcript.setTimestamps(enabled);
|
|
112
|
+
this.patch(null, { timestamps: enabled });
|
|
113
|
+
}
|
|
114
|
+
async start(source, request) {
|
|
115
|
+
if (this.disposed) {
|
|
116
|
+
request
|
|
117
|
+
?.then((stream) => stream.getTracks().forEach((track) => track.stop()))
|
|
118
|
+
.catch(() => {});
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
this.input(source);
|
|
122
|
+
const stopping = this.stop(source);
|
|
123
|
+
const generation = this.generations.get(source);
|
|
124
|
+
await stopping;
|
|
125
|
+
if (this.disposed || this.generations.get(source) !== generation) {
|
|
126
|
+
request
|
|
127
|
+
?.then((stream) => stream.getTracks().forEach((track) => track.stop()))
|
|
128
|
+
.catch(() => {});
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
this.patch(null, { active: true });
|
|
132
|
+
const abort = new AbortController();
|
|
133
|
+
const job = { abort };
|
|
134
|
+
this.jobs.set(source, job);
|
|
135
|
+
const valid = () => this.jobs.get(source) === job && !abort.signal.aborted;
|
|
136
|
+
this.patch(source, { starting: true, error: null });
|
|
137
|
+
try {
|
|
138
|
+
const settings = this.settings();
|
|
139
|
+
if (settings.recognitionEngine === 'browser') throw new Error(this.t('httpRequired'));
|
|
140
|
+
const { meter, engine } = this.createSource(source, settings);
|
|
141
|
+
const input = this.input(source);
|
|
142
|
+
input.meter = meter;
|
|
143
|
+
input.updateSettings({ microphoneEnabled: true });
|
|
144
|
+
this.applySettings(settings);
|
|
145
|
+
input.patch({ conversation: true, starting: true, error: null });
|
|
146
|
+
Object.assign(job, { meter, engine });
|
|
147
|
+
if (source === 'shared') meter.provide(request);
|
|
148
|
+
meter.deviceId = settings.inputDeviceId;
|
|
149
|
+
const capability = await engine.capability();
|
|
150
|
+
if (!valid()) return;
|
|
151
|
+
if (!capability.supported) throw new Error(capability.reason || this.t('unavailable'));
|
|
152
|
+
if (!(await meter.start({ signal: abort.signal }))) {
|
|
153
|
+
if (valid()) await this.stop(source);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
if (!valid()) return;
|
|
157
|
+
this.transcript.setActive(source, true);
|
|
158
|
+
for (const track of meter.stream?.getTracks() || [])
|
|
159
|
+
track.addEventListener(
|
|
160
|
+
'ended',
|
|
161
|
+
() => {
|
|
162
|
+
if (valid()) void this.stop(source);
|
|
163
|
+
},
|
|
164
|
+
{ once: true },
|
|
165
|
+
);
|
|
166
|
+
await engine.start({
|
|
167
|
+
lang: settings.recognitionLang,
|
|
168
|
+
signal: abort.signal,
|
|
169
|
+
onResult: ({ final, interim, startedAt }) => {
|
|
170
|
+
if (!valid()) return;
|
|
171
|
+
const accepted = this.onSpeech?.(source, final || interim) !== false;
|
|
172
|
+
if (accepted) input.onResult({ final, interim, startedAt });
|
|
173
|
+
else if (final) input.patch({ recognizing: false });
|
|
174
|
+
if (final && !interim) {
|
|
175
|
+
this.onActivity?.(source, false);
|
|
176
|
+
input.refreshDeliveryReadiness();
|
|
177
|
+
}
|
|
178
|
+
},
|
|
179
|
+
onActivity: (active) => {
|
|
180
|
+
if (!valid()) return;
|
|
181
|
+
if (active) input.cancelAutoSend({ preserveIntent: true });
|
|
182
|
+
input.patch({ recognizing: active });
|
|
183
|
+
this.onActivity?.(source, active);
|
|
184
|
+
input.refreshDeliveryReadiness();
|
|
185
|
+
},
|
|
186
|
+
onProcessingChange: ({ pending }) => {
|
|
187
|
+
if (!valid()) return;
|
|
188
|
+
input.patch({ pendingTranscriptions: pending });
|
|
189
|
+
this.patch(source, { pending });
|
|
190
|
+
if (pending) input.cancelAutoSend({ preserveIntent: true });
|
|
191
|
+
else if (!input.getSnapshot().muted) input.maybeScheduleAutoSend();
|
|
192
|
+
},
|
|
193
|
+
onError: (error) => {
|
|
194
|
+
if (valid()) {
|
|
195
|
+
this.patch(source, { error: this.t('failed') });
|
|
196
|
+
void this.stop(source);
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
if (valid()) {
|
|
201
|
+
input.patch({ starting: false, listening: true });
|
|
202
|
+
this.patch(source, { starting: false, listening: true });
|
|
203
|
+
}
|
|
204
|
+
} catch (error) {
|
|
205
|
+
if (valid()) {
|
|
206
|
+
this.patch(source, {
|
|
207
|
+
error:
|
|
208
|
+
source === 'shared' && error.message === 'No shared audio track was returned.'
|
|
209
|
+
? this.t('noAudio')
|
|
210
|
+
: error.message === this.t('httpRequired')
|
|
211
|
+
? error.message
|
|
212
|
+
: this.t('failed'),
|
|
213
|
+
});
|
|
214
|
+
await this.stop(source);
|
|
215
|
+
}
|
|
216
|
+
} finally {
|
|
217
|
+
if (!valid()) {
|
|
218
|
+
await job.engine?.stop();
|
|
219
|
+
await job.meter?.stop();
|
|
220
|
+
}
|
|
221
|
+
if (source === 'shared' && !job.meter?.stream)
|
|
222
|
+
request
|
|
223
|
+
?.then((stream) => stream.getTracks().forEach((track) => track.stop()))
|
|
224
|
+
.catch(() => {});
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
async stop(source) {
|
|
228
|
+
this.generations.set(source, (this.generations.get(source) || 0) + 1);
|
|
229
|
+
const job = this.jobs.get(source);
|
|
230
|
+
this.jobs.delete(source);
|
|
231
|
+
const input = this.inputs.get(source);
|
|
232
|
+
input?.cancelAutoSend();
|
|
233
|
+
input?.patch({ conversation: false, starting: false, listening: false, recognizing: false, pendingTranscriptions: 0 });
|
|
234
|
+
this.onActivity?.(source, false);
|
|
235
|
+
job?.abort.abort();
|
|
236
|
+
this.transcript.setActive(source, false);
|
|
237
|
+
this.patch(source, { starting: false, listening: false, pending: 0 });
|
|
238
|
+
await job?.engine?.stop();
|
|
239
|
+
await job?.meter?.stop();
|
|
240
|
+
}
|
|
241
|
+
async end() {
|
|
242
|
+
await Promise.all(['microphone', 'shared'].map((source) => this.stop(source)));
|
|
243
|
+
this.transcript.reset();
|
|
244
|
+
this.patch(null, { active: false });
|
|
245
|
+
}
|
|
246
|
+
async dispose() {
|
|
247
|
+
this.disposed = true;
|
|
248
|
+
await this.end();
|
|
249
|
+
for (const input of this.inputs.values()) {
|
|
250
|
+
input.cancelAutoSend();
|
|
251
|
+
input.disposed = true;
|
|
252
|
+
input.listeners.clear();
|
|
253
|
+
}
|
|
254
|
+
this.listeners.clear();
|
|
255
|
+
}
|
|
256
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export type MeetingSource = 'microphone' | 'shared';
|
|
2
|
+
|
|
3
|
+
/** One composer writer; independent recognizers deliver final chunks in arrival order. */
|
|
4
|
+
export class MeetingTranscript {
|
|
5
|
+
private active = new Set<MeetingSource>();
|
|
6
|
+
private last: MeetingSource | null = null;
|
|
7
|
+
private expected: string | null = null;
|
|
8
|
+
private timestamps = false;
|
|
9
|
+
private sourceTimestamps = new Map<MeetingSource, boolean>();
|
|
10
|
+
setSourceTimestamps(source: MeetingSource, enabled: boolean) {
|
|
11
|
+
if (this.sourceTimestamps.get(source) === enabled) return;
|
|
12
|
+
this.sourceTimestamps.set(source, enabled);
|
|
13
|
+
this.last = null;
|
|
14
|
+
}
|
|
15
|
+
setTimestamps(enabled: boolean) {
|
|
16
|
+
this.timestamps = enabled;
|
|
17
|
+
this.last = null;
|
|
18
|
+
}
|
|
19
|
+
constructor(private composer: { getDraft(): string; setDraft(text: string): void }) {}
|
|
20
|
+
setActive(source: MeetingSource, enabled: boolean) {
|
|
21
|
+
if (enabled) this.active.add(source);
|
|
22
|
+
else this.active.delete(source);
|
|
23
|
+
this.last = null;
|
|
24
|
+
}
|
|
25
|
+
append(source: MeetingSource, text: string, startedAt?: number) {
|
|
26
|
+
const chunk = text.trim();
|
|
27
|
+
if (!chunk || !this.active.has(source)) return;
|
|
28
|
+
const draft = this.composer.getDraft();
|
|
29
|
+
if (draft !== this.expected) this.last = null;
|
|
30
|
+
const labelled = this.active.size > 1;
|
|
31
|
+
const timestamps = this.sourceTimestamps.get(source) ?? this.timestamps;
|
|
32
|
+
const changed = (labelled || timestamps) && this.last !== source;
|
|
33
|
+
const date = new Date(Number.isFinite(startedAt) ? startedAt! : Date.now());
|
|
34
|
+
const pad = (value: number) => String(value).padStart(2, '0');
|
|
35
|
+
const stamp =
|
|
36
|
+
timestamps && changed
|
|
37
|
+
? '[' +
|
|
38
|
+
date.getFullYear() +
|
|
39
|
+
'/' +
|
|
40
|
+
pad(date.getMonth() + 1) +
|
|
41
|
+
'/' +
|
|
42
|
+
pad(date.getDate()) +
|
|
43
|
+
' ' +
|
|
44
|
+
pad(date.getHours()) +
|
|
45
|
+
':' +
|
|
46
|
+
pad(date.getMinutes()) +
|
|
47
|
+
':' +
|
|
48
|
+
pad(date.getSeconds()) +
|
|
49
|
+
'] '
|
|
50
|
+
: '';
|
|
51
|
+
const prefix = changed ? stamp + (labelled ? (source === 'microphone' ? 'Me: ' : 'Them: ') : '') : '';
|
|
52
|
+
const separator = draft ? (changed ? '\n\n' : '\n') : '';
|
|
53
|
+
const next = draft + separator + prefix + chunk;
|
|
54
|
+
this.composer.setDraft(next);
|
|
55
|
+
this.expected = next;
|
|
56
|
+
this.last = labelled || timestamps ? source : null;
|
|
57
|
+
}
|
|
58
|
+
reset() {
|
|
59
|
+
this.active.clear();
|
|
60
|
+
this.last = null;
|
|
61
|
+
this.expected = null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -2,4 +2,4 @@
|
|
|
2
2
|
|
|
3
3
|
`core` owns policy and reusable voice-domain state, not DSH slot/route registration or provider-specific presentation. `coordinator.ts` serializes input transitions, owns conversation state and speech queues, and invalidates stale async work with generations. `ownership.ts` arbitrates microphone and speaker use between sessions and Settings self-tests. `transcript.ts` enforces append-only final recognition text (interim hypotheses never own or rewrite composer content), `microphone.ts` manages capture and activity, `filters.ts` handles voice commands and speech text filtering, and `settings.ts` normalizes persisted values and defaults.
|
|
4
4
|
|
|
5
|
-
The `qwen/` subtree contains shared Qwen host configuration and its settings UI; recognition and speaking adapters live in their respective modules. Keep the client/host dependency graphs separate. Preserve the persisted `dsh-live-voice.settings` key and normalization compatibility. Distinguish capture, recognition, playback, and generation; do not conflate pausing with cancellation or resume obsolete speech after a new turn. Consult [voice lifecycle](
|
|
5
|
+
The `qwen/` subtree contains shared Qwen host configuration and its settings UI; recognition and speaking adapters live in their respective modules. Keep the client/host dependency graphs separate. Preserve the persisted `dsh-live-voice.settings` key and normalization compatibility. Distinguish capture, recognition, playback, and generation; do not conflate pausing with cancellation or resume obsolete speech after a new turn. Consult [voice lifecycle](../../../../../docs/VOICE-LIFECYCLE.md) for expected transitions.
|