dsh-live-voice 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -14
- package/lib/client.js +6474 -3396
- package/lib/server.js +29 -6
- package/package.json +9 -30
- package/src/app/AGENTS.md +1 -1
- package/src/app/ARCHITECTURE.md +1 -1
- package/src/app/client/apply.tsx +242 -35
- package/src/app/client/composerSelection.ts +92 -0
- package/src/app/client/i18n/catalogs/base.ts +35 -0
- package/src/app/client/i18n/catalogs/en.ts +43 -3
- package/src/app/client/i18n/catalogs/es.ts +51 -11
- package/src/app/client/i18n/catalogs/fr.ts +52 -11
- package/src/app/client/i18n/catalogs/hi.ts +50 -11
- package/src/app/client/i18n/catalogs/pt-BR.ts +52 -11
- package/src/app/client/i18n/catalogs/zh.ts +50 -11
- package/src/modules/AGENTS.md +1 -1
- package/src/modules/conversation/ARCHITECTURE.md +6 -2
- package/src/modules/conversation/components/AutoPlaybackToggle.tsx +31 -0
- package/src/modules/conversation/components/ConversationControls.tsx +23 -8
- package/src/modules/conversation/components/ConversationStatusBar.tsx +91 -93
- package/src/modules/conversation/components/DeliveryModeButton.tsx +3 -3
- package/src/modules/conversation/components/MeetingControls.tsx +40 -0
- package/src/modules/conversation/components/MicrophoneButton.tsx +15 -2
- package/src/modules/conversation/components/PlaybackControls.tsx +20 -5
- package/src/modules/conversation/components/RecognitionBar.tsx +14 -0
- package/src/modules/conversation/components/ScrollingSpeechCaption.tsx +100 -0
- package/src/modules/conversation/components/SpeechStatusBar.tsx +99 -0
- package/src/modules/conversation/components/Waveform.tsx +8 -3
- package/src/modules/conversation/components/conversationStatus.ts +1 -3
- package/src/modules/conversation/components/createConversationComponents.tsx +1 -1
- package/src/modules/conversation/components/index.ts +1 -0
- package/src/modules/conversation/components/speechCaption.ts +27 -0
- package/src/modules/conversation/models/chat.ts +15 -2
- package/src/modules/conversation/models/meeting.ts +256 -0
- package/src/modules/conversation/models/meetingTranscript.ts +63 -0
- package/src/modules/core/ARCHITECTURE.md +1 -1
- package/src/modules/core/coordinator.ts +353 -81
- package/src/modules/core/developerExtension.ts +26 -0
- package/src/modules/core/diagnostics.ts +153 -0
- package/src/modules/core/filters.ts +20 -15
- package/src/modules/core/markdownSpeech.ts +154 -0
- package/src/modules/core/settings.ts +17 -13
- package/src/modules/core/sharedAudio.ts +88 -0
- package/src/modules/core/speechDefaults.ts +21 -0
- package/src/modules/recognition/engines/browser/BrowserRecognitionEngine.ts +8 -1
- package/src/modules/recognition/engines/whisper/WhisperRecognitionEngine.ts +10 -3
- package/src/modules/settings/components/LiveVoiceSettings.tsx +17 -3
- package/src/modules/settings/components/SettingsHeader.tsx +1 -1
- package/src/modules/settings/models/settingsStorage.ts +11 -3
- package/src/modules/settings/sections/conversation/ConversationDelaySettings.tsx +8 -3
- package/src/modules/settings/sections/conversation/DeliverySettings.tsx +25 -2
- package/src/modules/settings/sections/recognition/RecognitionEngineSettings.tsx +2 -1
- package/src/modules/settings/sections/speak/SpeechEngineSettings.tsx +12 -1
- package/src/modules/speak/engines/audio/HostAudioEngine.ts +19 -2
- package/src/modules/speak/engines/browser/BrowserSpeakingEngine.ts +33 -2
- package/src/shared/design-system/buttons/ToggleButton.tsx +8 -1
- package/src/shared/design-system/icons/icons.ts +6 -0
- package/src/styles/index.ts +13 -3
- package/CHANGELOG.md +0 -240
- package/DEVELOPMENT.md +0 -83
- package/PLAN.md +0 -296
- package/docs/ARCHITECTURE.md +0 -33
- package/docs/CHOOSING-AN-ENGINE.md +0 -120
- package/docs/CONFIGURATION.md +0 -163
- package/docs/REVIEW.md +0 -35
- package/docs/VOICE-LIFECYCLE.md +0 -63
- package/scripts/build.ts +0 -62
- package/scripts/check-dist.mjs +0 -38
- package/scripts/preview-ui.ts +0 -31
- package/scripts/probe-browser.ts +0 -46
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ComponentType } from 'react';
|
|
2
|
+
export const DEVELOPER_KEY = Symbol.for('dsh-live-voice.developer.v1');
|
|
3
|
+
export const DEVELOPER_CHANGED = 'dsh-live-voice:developer-changed';
|
|
4
|
+
export interface DeveloperExtension {
|
|
5
|
+
version: 1;
|
|
6
|
+
label(): string;
|
|
7
|
+
component: ComponentType;
|
|
8
|
+
}
|
|
9
|
+
export function readDeveloperExtension(target: any = globalThis.window): DeveloperExtension | null {
|
|
10
|
+
const extension = target?.[DEVELOPER_KEY];
|
|
11
|
+
return extension?.version === 1 ? extension : null;
|
|
12
|
+
}
|
|
13
|
+
export function subscribeDeveloperExtension(listener: () => void, target: any = globalThis.window) {
|
|
14
|
+
target?.addEventListener(DEVELOPER_CHANGED, listener);
|
|
15
|
+
return () => target?.removeEventListener(DEVELOPER_CHANGED, listener);
|
|
16
|
+
}
|
|
17
|
+
export function publishDeveloperExtension(target: any, extension: DeveloperExtension) {
|
|
18
|
+
const ChangedEvent = target.Event ?? Event;
|
|
19
|
+
target[DEVELOPER_KEY] = extension;
|
|
20
|
+
target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
|
|
21
|
+
return () => {
|
|
22
|
+
if (target[DEVELOPER_KEY] !== extension) return;
|
|
23
|
+
delete target[DEVELOPER_KEY];
|
|
24
|
+
target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
|
|
25
|
+
};
|
|
26
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/** Versioned read-only boundary; debugger UI is never imported by the runtime. */
|
|
2
|
+
export const DIAGNOSTIC_KEY = Symbol.for('dsh-live-voice.diagnostics.v1');
|
|
3
|
+
export const DIAGNOSTIC_CHANGED = 'dsh-live-voice:diagnostics-changed';
|
|
4
|
+
export type DiagnosticValue =
|
|
5
|
+
null | boolean | number | string | DiagnosticValue[] | { [key: string]: DiagnosticValue };
|
|
6
|
+
export interface DiagnosticSource {
|
|
7
|
+
version: 1;
|
|
8
|
+
read(options?: { includeContent?: boolean }): DiagnosticValue;
|
|
9
|
+
subscribe(listener: () => void): () => void;
|
|
10
|
+
}
|
|
11
|
+
export function publishDiagnosticSource(target: any, source: DiagnosticSource) {
|
|
12
|
+
target[DIAGNOSTIC_KEY] = source;
|
|
13
|
+
const ChangedEvent = target.Event ?? Event;
|
|
14
|
+
target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
|
|
15
|
+
return () => {
|
|
16
|
+
if (target[DIAGNOSTIC_KEY] !== source) return;
|
|
17
|
+
delete target[DIAGNOSTIC_KEY];
|
|
18
|
+
target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const number = (value: unknown) =>
|
|
22
|
+
typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
23
|
+
/** Explicit projection excludes audio, functions, resources and free-form errors. */
|
|
24
|
+
export function readCoordinatorDiagnostics(
|
|
25
|
+
controller: any,
|
|
26
|
+
includeContent = false,
|
|
27
|
+
): DiagnosticValue {
|
|
28
|
+
const state = controller.getSnapshot();
|
|
29
|
+
const recognition = controller.recognition?.session;
|
|
30
|
+
const settings: Record<string, DiagnosticValue> = {};
|
|
31
|
+
const publicSettings = new Set([
|
|
32
|
+
'engine',
|
|
33
|
+
'recognitionEngine',
|
|
34
|
+
'voiceDetectionPreset',
|
|
35
|
+
'sendingMode',
|
|
36
|
+
'dshBusyEnterBehavior',
|
|
37
|
+
'mode',
|
|
38
|
+
'lang',
|
|
39
|
+
'recognitionLang',
|
|
40
|
+
]);
|
|
41
|
+
for (const [key, value] of Object.entries(state.settings ?? {})) {
|
|
42
|
+
settings[key] =
|
|
43
|
+
typeof value === 'string'
|
|
44
|
+
? includeContent || publicSettings.has(key)
|
|
45
|
+
? value
|
|
46
|
+
: { characters: value.length }
|
|
47
|
+
: typeof value === 'boolean' || typeof value === 'number'
|
|
48
|
+
? value
|
|
49
|
+
: null;
|
|
50
|
+
}
|
|
51
|
+
const item = (entry: any): DiagnosticValue => ({
|
|
52
|
+
messageId: entry.id == null ? null : String(entry.id),
|
|
53
|
+
manual: entry.manual === true,
|
|
54
|
+
characters: entry.text?.length ?? 0,
|
|
55
|
+
...(includeContent ? { text: String(entry.text ?? '') } : {}),
|
|
56
|
+
preparing: Boolean(entry.preparing),
|
|
57
|
+
prepared: Boolean(entry.prepared),
|
|
58
|
+
failed: Boolean(entry.prepareError),
|
|
59
|
+
});
|
|
60
|
+
return {
|
|
61
|
+
settings,
|
|
62
|
+
lifecycle: {
|
|
63
|
+
disposed: Boolean(controller.disposed),
|
|
64
|
+
conversation: Boolean(state.conversation),
|
|
65
|
+
},
|
|
66
|
+
composer: {
|
|
67
|
+
characters: controller.composer?.getDraft?.()?.length ?? 0,
|
|
68
|
+
answeringQuestion: Boolean(state.answeringQuestion),
|
|
69
|
+
...(includeContent ? { draft: controller.composer?.getDraft?.() ?? '' } : {}),
|
|
70
|
+
},
|
|
71
|
+
capture: {
|
|
72
|
+
starting: Boolean(state.starting),
|
|
73
|
+
listening: Boolean(state.listening),
|
|
74
|
+
muted: Boolean(state.muted),
|
|
75
|
+
generation: number(controller.epoch),
|
|
76
|
+
level: number(controller.meter?.level?.()),
|
|
77
|
+
releaseFailed: Boolean(controller.inputReleaseError),
|
|
78
|
+
},
|
|
79
|
+
vad: {
|
|
80
|
+
source: state.settings?.recognitionEngine === 'browser' ? 'native' : 'plugin',
|
|
81
|
+
activity: Boolean(state.recognizing),
|
|
82
|
+
threshold: recognition?.chunks ? 0.012 : null,
|
|
83
|
+
silenceMs:
|
|
84
|
+
recognition?.chunks && controller.meter?.context?.sampleRate
|
|
85
|
+
? (recognition.silence / controller.meter.context.sampleRate) * 1000
|
|
86
|
+
: null,
|
|
87
|
+
},
|
|
88
|
+
chunker: {
|
|
89
|
+
available: Boolean(recognition?.chunks),
|
|
90
|
+
bufferedSamples: number(recognition?.samples),
|
|
91
|
+
bufferedChunks: recognition?.chunks?.length ?? null,
|
|
92
|
+
containsSpeech: recognition?.voiced ?? null,
|
|
93
|
+
},
|
|
94
|
+
recognition: {
|
|
95
|
+
pendingTranscriptions: number(state.pendingTranscriptions),
|
|
96
|
+
activeRequest: Boolean(recognition?.activeRequest),
|
|
97
|
+
queue: (recognition?.transcriptionQueue ?? []).map((chunk: any) => ({
|
|
98
|
+
samples: (chunk.samples ?? chunk).length,
|
|
99
|
+
durationMs: controller.meter?.context?.sampleRate
|
|
100
|
+
? ((chunk.samples ?? chunk).length / controller.meter.context.sampleRate) * 1000
|
|
101
|
+
: null,
|
|
102
|
+
})),
|
|
103
|
+
},
|
|
104
|
+
delivery: {
|
|
105
|
+
autoSendAt: number(state.autoSendAt),
|
|
106
|
+
countdownActive: controller.autoSendTimer != null,
|
|
107
|
+
holdToTalkRelease: Boolean(controller.holdToTalkRelease),
|
|
108
|
+
},
|
|
109
|
+
speech: {
|
|
110
|
+
configuredEngine: state.settings?.engine ?? null,
|
|
111
|
+
activeEngine: state.speechEngine ?? null,
|
|
112
|
+
activeEndpoint: controller.engines?.[state.speechEngine]?.endpoint ?? null,
|
|
113
|
+
speaking: Boolean(state.speaking),
|
|
114
|
+
paused: Boolean(state.paused),
|
|
115
|
+
activeMessageId: state.activeMessageId == null ? null : String(state.activeMessageId),
|
|
116
|
+
generation: number(controller.speechEpoch),
|
|
117
|
+
controlGeneration: number(controller.controlEpoch),
|
|
118
|
+
remaining: number(state.speechSegmentsRemaining),
|
|
119
|
+
notBefore: number(controller.assistantSpeechNotBefore),
|
|
120
|
+
waitingForActivity: controller.assistantSpeechNotBefore === Infinity,
|
|
121
|
+
stopFailed: Boolean(controller.speechStopError),
|
|
122
|
+
queue: (controller.queue ?? []).map(item),
|
|
123
|
+
activeItem: controller.activeSpeechItem ? item(controller.activeSpeechItem) : null,
|
|
124
|
+
prefetchCount: controller.prefetchItems?.size ?? 0,
|
|
125
|
+
consumed: Object.fromEntries(controller.consumed ?? []),
|
|
126
|
+
unfinished: [...(controller.unfinished ?? [])].map(String),
|
|
127
|
+
suppressed: [...(controller.suppressed ?? [])].map(String),
|
|
128
|
+
},
|
|
129
|
+
interruption: {
|
|
130
|
+
transcriptConfirmed: Boolean(controller.interruptionTranscriptConfirmed),
|
|
131
|
+
pausedSpeech: Boolean(controller.interruptionPausedSpeech),
|
|
132
|
+
timerActive: controller.interruptionTimer != null,
|
|
133
|
+
},
|
|
134
|
+
capabilities: Object.fromEntries(
|
|
135
|
+
Object.entries(state.capabilities ?? {}).map(([key, cap]: [string, any]) => [
|
|
136
|
+
key,
|
|
137
|
+
cap
|
|
138
|
+
? {
|
|
139
|
+
supported: cap.supported === true,
|
|
140
|
+
local: cap.local ?? null,
|
|
141
|
+
pause: cap.pause ?? null,
|
|
142
|
+
resume: cap.resume ?? null,
|
|
143
|
+
hasReason: Boolean(cap.reason),
|
|
144
|
+
}
|
|
145
|
+
: null,
|
|
146
|
+
]),
|
|
147
|
+
),
|
|
148
|
+
error: {
|
|
149
|
+
present: Boolean(state.error),
|
|
150
|
+
...(includeContent ? { message: state.error ?? null } : {}),
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
// @ts-nocheck
|
|
2
|
+
import { markdownSpeech } from './markdownSpeech.js';
|
|
3
|
+
export { hasIncompleteSpeechMarkdown } from './markdownSpeech.js';
|
|
2
4
|
const words = (text) =>
|
|
3
5
|
String(text || '')
|
|
4
6
|
.trim()
|
|
@@ -41,23 +43,26 @@ export function splitSpeechOutput(text, options = {}) {
|
|
|
41
43
|
.filter(Boolean);
|
|
42
44
|
}
|
|
43
45
|
|
|
46
|
+
/** Split only the initial item at sentence punctuation or colons, never commas. */
|
|
47
|
+
export function splitInitialSpeechItem(text) {
|
|
48
|
+
const result = [];
|
|
49
|
+
let start = 0;
|
|
50
|
+
// Require whitespace/end after punctuation to preserve decimals, URLs and file names.
|
|
51
|
+
for (const match of text.matchAll(/[.!?:]+(?:["”’)]*)?(?=\s|$)/gu)) {
|
|
52
|
+
const end = match.index + match[0].length;
|
|
53
|
+
const part = text.slice(start, end).trim();
|
|
54
|
+
if (part) result.push(part);
|
|
55
|
+
start = end;
|
|
56
|
+
}
|
|
57
|
+
const tail = text.slice(start).trim();
|
|
58
|
+
if (tail) result.push(tail);
|
|
59
|
+
return result.length ? result : [text];
|
|
60
|
+
}
|
|
61
|
+
|
|
44
62
|
export function hasUnclosedCodeFence(text) {
|
|
45
63
|
return (String(text || '').match(/```/g) || []).length % 2 === 1;
|
|
46
64
|
}
|
|
47
65
|
|
|
48
|
-
export function filterSpeechOutput(
|
|
49
|
-
text,
|
|
50
|
-
{
|
|
51
|
-
filterCodeBlocks = true,
|
|
52
|
-
codeBlockMaxLines = 5,
|
|
53
|
-
codeBlockNotice = 'Look the code on out conversation',
|
|
54
|
-
} = {},
|
|
55
|
-
) {
|
|
56
|
-
const source = String(text || '');
|
|
57
|
-
if (!filterCodeBlocks) return source;
|
|
58
|
-
return source.replace(/```[^\n]*\n?([\s\S]*?)```/g, (block, body) => {
|
|
59
|
-
const normalized = body.replace(/\n$/, '');
|
|
60
|
-
const lineCount = normalized ? normalized.split(/\r?\n/).length : 0;
|
|
61
|
-
return lineCount <= codeBlockMaxLines ? normalized : codeBlockNotice;
|
|
62
|
-
});
|
|
66
|
+
export function filterSpeechOutput(text, options = {}) {
|
|
67
|
+
return markdownSpeech(text, options);
|
|
63
68
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
import { marked } from 'marked';
|
|
3
|
+
import { resolveCodeNotice } from './speechDefaults.js';
|
|
4
|
+
|
|
5
|
+
// Spoken phrases follow the TTS language, independently of the interface locale.
|
|
6
|
+
const phrases = {
|
|
7
|
+
en: [
|
|
8
|
+
'Link to',
|
|
9
|
+
'Image:',
|
|
10
|
+
'Table with columns:',
|
|
11
|
+
'Checked:',
|
|
12
|
+
'Unchecked:',
|
|
13
|
+
'line',
|
|
14
|
+
'lines',
|
|
15
|
+
'to',
|
|
16
|
+
'identifier',
|
|
17
|
+
],
|
|
18
|
+
pt: [
|
|
19
|
+
'Link para',
|
|
20
|
+
'Imagem:',
|
|
21
|
+
'Tabela com colunas:',
|
|
22
|
+
'Marcado:',
|
|
23
|
+
'Desmarcado:',
|
|
24
|
+
'linha',
|
|
25
|
+
'linhas',
|
|
26
|
+
'a',
|
|
27
|
+
'identificador',
|
|
28
|
+
],
|
|
29
|
+
es: [
|
|
30
|
+
'Enlace a',
|
|
31
|
+
'Imagen:',
|
|
32
|
+
'Tabla con columnas:',
|
|
33
|
+
'Marcado:',
|
|
34
|
+
'Desmarcado:',
|
|
35
|
+
'línea',
|
|
36
|
+
'líneas',
|
|
37
|
+
'a',
|
|
38
|
+
'identificador',
|
|
39
|
+
],
|
|
40
|
+
fr: [
|
|
41
|
+
'Lien vers',
|
|
42
|
+
'Image :',
|
|
43
|
+
'Tableau avec les colonnes :',
|
|
44
|
+
'Coché :',
|
|
45
|
+
'Décoché :',
|
|
46
|
+
'ligne',
|
|
47
|
+
'lignes',
|
|
48
|
+
'à',
|
|
49
|
+
'identifiant',
|
|
50
|
+
],
|
|
51
|
+
hi: [
|
|
52
|
+
'लिंक',
|
|
53
|
+
'चित्र:',
|
|
54
|
+
'तालिका के स्तंभ:',
|
|
55
|
+
'चिह्नित:',
|
|
56
|
+
'अचिह्नित:',
|
|
57
|
+
'पंक्ति',
|
|
58
|
+
'पंक्तियाँ',
|
|
59
|
+
'से',
|
|
60
|
+
'पहचानकर्ता',
|
|
61
|
+
],
|
|
62
|
+
zh: ['链接到', '图片:', '表格列:', '已勾选:', '未勾选:', '行', '行', '至', '标识符'],
|
|
63
|
+
};
|
|
64
|
+
export function markdownSpeech(source, options = {}) {
|
|
65
|
+
const p = phrases[String(options.lang || 'en').split('-')[0]] || phrases.en;
|
|
66
|
+
const host = (url) => {
|
|
67
|
+
try {
|
|
68
|
+
return new URL(url).hostname;
|
|
69
|
+
} catch {
|
|
70
|
+
return '';
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
const plain = (text) =>
|
|
74
|
+
String(text)
|
|
75
|
+
.replace(/https?:\/\/[^\s<>]+/gu, (url) => p[0] + ' ' + host(url))
|
|
76
|
+
.replace(
|
|
77
|
+
/(?:[A-Za-z]:[\\/]|\.{0,2}\/)?(?:[\w.@~-]+[\\/])+([\w.@~-]+\.[\w]+)((?::\d+(?:-\d+)?)|(?:#L\d+(?:-L?\d+)?))?/gu,
|
|
78
|
+
(_, name, loc = '') => name + loc,
|
|
79
|
+
)
|
|
80
|
+
.replace(
|
|
81
|
+
/([\w.@~-]+\.[\w]+)(?::(\d+)(?:-(\d+))?|#L(\d+)(?:-L?(\d+))?)/gu,
|
|
82
|
+
(_, name, a, b, c, d) =>
|
|
83
|
+
name +
|
|
84
|
+
', ' +
|
|
85
|
+
(b || d ? p[6] : p[5]) +
|
|
86
|
+
' ' +
|
|
87
|
+
(a || c) +
|
|
88
|
+
(b || d ? ' ' + p[7] + ' ' + (b || d) : ''),
|
|
89
|
+
)
|
|
90
|
+
.replace(
|
|
91
|
+
/\b(?:[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}|[a-f\d]{24,})\b/giu,
|
|
92
|
+
p[8],
|
|
93
|
+
);
|
|
94
|
+
const inline = (tokens = []) =>
|
|
95
|
+
tokens
|
|
96
|
+
.map((t) => {
|
|
97
|
+
if (t.type === 'link') {
|
|
98
|
+
const name = t.text === t.href ? host(t.href) : inline(t.tokens);
|
|
99
|
+
return p[0] + ' ' + (name || host(t.href) || plain(t.href));
|
|
100
|
+
}
|
|
101
|
+
if (t.type === 'image') return t.text ? p[1] + ' ' + plain(t.text) : '';
|
|
102
|
+
if (t.type === 'html') return '';
|
|
103
|
+
if (t.type === 'br') return '\n';
|
|
104
|
+
if (t.tokens) return inline(t.tokens);
|
|
105
|
+
return plain(t.text ?? t.raw ?? '');
|
|
106
|
+
})
|
|
107
|
+
.join('');
|
|
108
|
+
const blocks = (tokens) =>
|
|
109
|
+
tokens
|
|
110
|
+
.map((t) => {
|
|
111
|
+
if (t.type === 'space') return t.raw;
|
|
112
|
+
if (t.type === 'hr' || t.type === 'def' || t.type === 'html') return '';
|
|
113
|
+
if (t.type === 'code')
|
|
114
|
+
return options.filterCodeBlocks !== false &&
|
|
115
|
+
t.text.split(/\r?\n/).length > (options.codeBlockMaxLines ?? 5)
|
|
116
|
+
? resolveCodeNotice(options.codeBlockNotice, options.lang)
|
|
117
|
+
: t.text;
|
|
118
|
+
if (t.type === 'blockquote') return blocks(t.tokens);
|
|
119
|
+
if (t.type === 'list')
|
|
120
|
+
return t.items
|
|
121
|
+
.map(
|
|
122
|
+
(item, i) =>
|
|
123
|
+
(item.task
|
|
124
|
+
? (item.checked ? p[3] : p[4]) + ' '
|
|
125
|
+
: t.ordered
|
|
126
|
+
? Number(t.start) + i + '. '
|
|
127
|
+
: '') + blocks(item.tokens).trim(),
|
|
128
|
+
)
|
|
129
|
+
.join('\n');
|
|
130
|
+
if (t.type === 'table')
|
|
131
|
+
return (
|
|
132
|
+
p[2] +
|
|
133
|
+
' ' +
|
|
134
|
+
t.header.map((c) => inline(c.tokens)).join(', ') +
|
|
135
|
+
'\n' +
|
|
136
|
+
t.rows.map((row) => row.map((c) => inline(c.tokens)).join(', ')).join('\n')
|
|
137
|
+
);
|
|
138
|
+
return t.tokens ? inline(t.tokens) : plain(t.text || '');
|
|
139
|
+
})
|
|
140
|
+
.reduce(
|
|
141
|
+
(result, value) =>
|
|
142
|
+
!value
|
|
143
|
+
? result
|
|
144
|
+
: result +
|
|
145
|
+
(result && !result.endsWith('\n') && !value.startsWith('\n') ? '\n' : '') +
|
|
146
|
+
value,
|
|
147
|
+
'',
|
|
148
|
+
);
|
|
149
|
+
return blocks(marked.lexer(String(source || ''), { gfm: true })).trim();
|
|
150
|
+
}
|
|
151
|
+
export function hasIncompleteSpeechMarkdown(source) {
|
|
152
|
+
if (/^[ \t]*\|/mu.test(source) || /\]\[[^\]]*\]/u.test(source)) return true;
|
|
153
|
+
return /!?\[[^\]]*$|!?\[[^\]]*\]\([^)]*$|!?\[[^\]]*\]\[[^\]]*\]/u.test(source);
|
|
154
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// @ts-nocheck
|
|
2
|
+
import { resolveCodeNotice } from './speechDefaults.js';
|
|
2
3
|
export const voiceDetectionPresets = Object.freeze({
|
|
3
4
|
short: Object.freeze({
|
|
4
5
|
silenceMs: 500,
|
|
@@ -34,6 +35,8 @@ export const defaultAgentVoiceContext = `Live Voice output is active. Your entir
|
|
|
34
35
|
Be concise and conversational. Lead with the answer or next action. Avoid unnecessary repetition, long preambles, dense lists, raw code, paths, and verbose status narration.
|
|
35
36
|
Do not narrate routine tool activity by default. If the user explicitly asks you to keep them informed while working, provide brief spoken progress updates only at meaningful milestones.`;
|
|
36
37
|
|
|
38
|
+
export const autoSendDelayOptions = Object.freeze([0.6, 0.8, 1, 2, 3, 4, 5, 6]);
|
|
39
|
+
|
|
37
40
|
export const defaultSettings = Object.freeze({
|
|
38
41
|
engine: 'browser',
|
|
39
42
|
recognitionEngine: 'browser',
|
|
@@ -43,14 +46,15 @@ export const defaultSettings = Object.freeze({
|
|
|
43
46
|
voiceDetectionCustomSilenceMs: 1000,
|
|
44
47
|
recognitionMaxUtteranceSeconds: 60,
|
|
45
48
|
microphoneEnabled: true,
|
|
46
|
-
holdToTalkEnabled:
|
|
49
|
+
holdToTalkEnabled: false,
|
|
47
50
|
announceAssistantMessages: true,
|
|
48
51
|
agentVoiceContextEnabled: true,
|
|
49
52
|
agentVoiceContext: defaultAgentVoiceContext,
|
|
50
53
|
interruptSpeechOnUserMessage: false,
|
|
51
54
|
sendingMode: 'manual',
|
|
55
|
+
dshBusyEnterBehavior: 'queue',
|
|
52
56
|
autoSendDelaySeconds: 4,
|
|
53
|
-
assistantSpeechDelaySeconds:
|
|
57
|
+
assistantSpeechDelaySeconds: 0,
|
|
54
58
|
mode: 'speaker',
|
|
55
59
|
lang: 'pt-BR',
|
|
56
60
|
recognitionLang: 'pt-BR',
|
|
@@ -69,7 +73,7 @@ export const defaultSettings = Object.freeze({
|
|
|
69
73
|
voiceCommandClear: 'clear all, clear message',
|
|
70
74
|
outputCodeFilterEnabled: true,
|
|
71
75
|
outputCodeMaxLines: 5,
|
|
72
|
-
outputCodeNotice:
|
|
76
|
+
outputCodeNotice: resolveCodeNotice(undefined, 'pt-BR'),
|
|
73
77
|
rate: 1,
|
|
74
78
|
segmentGapMs: 400,
|
|
75
79
|
});
|
|
@@ -153,22 +157,22 @@ export function normalizeSettings(value) {
|
|
|
153
157
|
typeof source.interruptSpeechOnUserMessage === 'boolean'
|
|
154
158
|
? source.interruptSpeechOnUserMessage
|
|
155
159
|
: defaultSettings.interruptSpeechOnUserMessage,
|
|
160
|
+
dshBusyEnterBehavior: ['queue', 'steer'].includes(source.dshBusyEnterBehavior)
|
|
161
|
+
? source.dshBusyEnterBehavior
|
|
162
|
+
: defaultSettings.dshBusyEnterBehavior,
|
|
156
163
|
sendingMode:
|
|
157
164
|
source.sendingMode === 'automatic'
|
|
158
165
|
? 'queue'
|
|
159
166
|
: ['manual', 'queue', 'steer'].includes(source.sendingMode)
|
|
160
167
|
? source.sendingMode
|
|
161
168
|
: defaultSettings.sendingMode,
|
|
162
|
-
autoSendDelaySeconds:
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
source.autoSendDelaySeconds <= 10
|
|
166
|
-
? source.autoSendDelaySeconds
|
|
167
|
-
: defaultSettings.autoSendDelaySeconds,
|
|
169
|
+
autoSendDelaySeconds: autoSendDelayOptions.includes(source.autoSendDelaySeconds)
|
|
170
|
+
? source.autoSendDelaySeconds
|
|
171
|
+
: defaultSettings.autoSendDelaySeconds,
|
|
168
172
|
assistantSpeechDelaySeconds:
|
|
169
173
|
Number.isInteger(source.assistantSpeechDelaySeconds) &&
|
|
170
|
-
source.assistantSpeechDelaySeconds >=
|
|
171
|
-
source.assistantSpeechDelaySeconds <=
|
|
174
|
+
source.assistantSpeechDelaySeconds >= 0 &&
|
|
175
|
+
source.assistantSpeechDelaySeconds <= 4
|
|
172
176
|
? source.assistantSpeechDelaySeconds
|
|
173
177
|
: defaultSettings.assistantSpeechDelaySeconds,
|
|
174
178
|
mode: ['speaker', 'headphones'].includes(source.mode) ? source.mode : defaultSettings.mode,
|
|
@@ -257,8 +261,8 @@ export function normalizeSettings(value) {
|
|
|
257
261
|
source.outputCodeNotice.trim() &&
|
|
258
262
|
source.outputCodeNotice.length <= 300 &&
|
|
259
263
|
!source.outputCodeNotice.includes('\0')
|
|
260
|
-
? source.outputCodeNotice.
|
|
261
|
-
: defaultSettings.outputCodeNotice,
|
|
264
|
+
? resolveCodeNotice(source.outputCodeNotice, source.lang || defaultSettings.lang)
|
|
265
|
+
: resolveCodeNotice(defaultSettings.outputCodeNotice, source.lang || defaultSettings.lang),
|
|
262
266
|
rate: Number.isFinite(source.rate) && source.rate >= 0.1 && source.rate <= 3 ? source.rate : 1,
|
|
263
267
|
segmentGapMs:
|
|
264
268
|
Number.isFinite(source.segmentGapMs) &&
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
import { MicrophoneMeter } from './microphone.js';
|
|
3
|
+
|
|
4
|
+
/** Display permission must be requested synchronously from a user gesture. */
|
|
5
|
+
export function requestSharedAudio(globals = globalThis) {
|
|
6
|
+
return globals.navigator.mediaDevices.getDisplayMedia({
|
|
7
|
+
video: true,
|
|
8
|
+
audio: true,
|
|
9
|
+
// Chromium hints exclude this application's own tab; unsupported browsers ignore them.
|
|
10
|
+
selfBrowserSurface: 'exclude',
|
|
11
|
+
restrictOwnAudio: true,
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export class SharedAudioMeter extends MicrophoneMeter {
|
|
16
|
+
constructor(globals = globalThis) {
|
|
17
|
+
super(globals);
|
|
18
|
+
this.pending = null;
|
|
19
|
+
}
|
|
20
|
+
provide(request) {
|
|
21
|
+
this.pending = Promise.resolve(request);
|
|
22
|
+
this.pending.catch(() => {});
|
|
23
|
+
}
|
|
24
|
+
async capability() {
|
|
25
|
+
return {
|
|
26
|
+
supported:
|
|
27
|
+
typeof this.g.navigator?.mediaDevices?.getDisplayMedia === 'function' &&
|
|
28
|
+
typeof (this.g.AudioContext || this.g.webkitAudioContext) === 'function',
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
async stop() {
|
|
32
|
+
const pending = this.pending;
|
|
33
|
+
this.pending = null;
|
|
34
|
+
pending?.then((stream) => stream.getTracks().forEach((track) => track.stop())).catch(() => {});
|
|
35
|
+
return super.stop();
|
|
36
|
+
}
|
|
37
|
+
async start({ signal } = {}) {
|
|
38
|
+
const pending = this.pending;
|
|
39
|
+
this.pending = null;
|
|
40
|
+
await super.stop();
|
|
41
|
+
if (!pending) throw new Error('Shared audio permission was not requested.');
|
|
42
|
+
const job = {
|
|
43
|
+
signal,
|
|
44
|
+
stream: null,
|
|
45
|
+
context: null,
|
|
46
|
+
source: null,
|
|
47
|
+
cancel: () => {
|
|
48
|
+
if (this.current === job) void this.stop();
|
|
49
|
+
},
|
|
50
|
+
cancelled: () => {},
|
|
51
|
+
};
|
|
52
|
+
this.current = job;
|
|
53
|
+
signal?.addEventListener('abort', job.cancel, { once: true });
|
|
54
|
+
try {
|
|
55
|
+
const stream = await pending;
|
|
56
|
+
job.stream = stream;
|
|
57
|
+
if (signal?.aborted || this.current !== job) {
|
|
58
|
+
this._dispose(job);
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
const tracks = stream.getAudioTracks();
|
|
62
|
+
if (!tracks.length) throw new Error('No shared audio track was returned.');
|
|
63
|
+
const AudioContext = this.g.AudioContext || this.g.webkitAudioContext;
|
|
64
|
+
job.context = new AudioContext();
|
|
65
|
+
// Exclude video and never route captured audio to audible output.
|
|
66
|
+
job.source = job.context.createMediaStreamSource(new this.g.MediaStream(tracks));
|
|
67
|
+
job.analyser = job.context.createAnalyser();
|
|
68
|
+
job.analyser.fftSize = 256;
|
|
69
|
+
job.source.connect(job.analyser);
|
|
70
|
+
job.samples = new Float32Array(job.analyser.fftSize);
|
|
71
|
+
this.stream = stream;
|
|
72
|
+
this.context = job.context;
|
|
73
|
+
this.source = job.source;
|
|
74
|
+
this.analyser = job.analyser;
|
|
75
|
+
this.samples = job.samples;
|
|
76
|
+
await job.context.resume();
|
|
77
|
+
return this.current === job;
|
|
78
|
+
} catch (error) {
|
|
79
|
+
if (this.current === job) {
|
|
80
|
+
this.current = null;
|
|
81
|
+
this._clear();
|
|
82
|
+
}
|
|
83
|
+
this._dispose(job);
|
|
84
|
+
if (signal?.aborted) return false;
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
const notices: Record<string, string> = {
|
|
2
|
+
en: 'You can see the code in our conversation history.',
|
|
3
|
+
pt: 'Você pode ver o código no histórico da nossa conversa.',
|
|
4
|
+
es: 'Puedes ver el código en el historial de nuestra conversación.',
|
|
5
|
+
fr: 'Vous pouvez voir le code dans l’historique de notre conversation.',
|
|
6
|
+
hi: 'आप हमारी बातचीत के इतिहास में कोड देख सकते हैं।',
|
|
7
|
+
zh: '你可以在我们的对话历史中查看代码。',
|
|
8
|
+
};
|
|
9
|
+
const defaults = new Set([
|
|
10
|
+
...Object.values(notices),
|
|
11
|
+
'Look the code on out conversation',
|
|
12
|
+
'You can see the code on our conversation history.',
|
|
13
|
+
'You can see the code on our conversation history',
|
|
14
|
+
]);
|
|
15
|
+
/** Default notices follow TTS language; explicit custom notices remain unchanged. */
|
|
16
|
+
export function resolveCodeNotice(value: unknown, lang = 'en') {
|
|
17
|
+
const text = typeof value === 'string' ? value.trim() : '';
|
|
18
|
+
return !text || defaults.has(text)
|
|
19
|
+
? notices[String(lang).toLowerCase().split('-')[0]] || notices.en
|
|
20
|
+
: text;
|
|
21
|
+
}
|
|
@@ -205,10 +205,13 @@ export class BrowserRecognitionEngine {
|
|
|
205
205
|
recognition.interimResults = true;
|
|
206
206
|
session.recognition = recognition;
|
|
207
207
|
const finals = new Set();
|
|
208
|
+
const starts = new Map();
|
|
209
|
+
let speechStartedAt = null;
|
|
208
210
|
const valid = () => this.session === session && session.recognition === recognition;
|
|
209
211
|
recognition.onspeechstart = () => {
|
|
210
212
|
if (!valid()) return;
|
|
211
213
|
session.restarts = 0;
|
|
214
|
+
speechStartedAt = Date.now();
|
|
212
215
|
this._activity(session, true);
|
|
213
216
|
};
|
|
214
217
|
recognition.onspeechend = () => {
|
|
@@ -216,20 +219,24 @@ export class BrowserRecognitionEngine {
|
|
|
216
219
|
};
|
|
217
220
|
recognition.onresult = (event) => {
|
|
218
221
|
if (!valid()) return;
|
|
222
|
+
let startedAt = null;
|
|
219
223
|
const interim = [],
|
|
220
224
|
final = [];
|
|
221
225
|
for (let i = 0; i < event.results.length; i++) {
|
|
222
226
|
const result = event.results[i];
|
|
223
227
|
const text = result[0]?.transcript ?? '';
|
|
228
|
+
if (!finals.has(i) && !starts.has(i) && text.trim())
|
|
229
|
+
starts.set(i, speechStartedAt ?? Date.now());
|
|
224
230
|
if (text.trim()) session.restarts = 0;
|
|
225
231
|
if (result.isFinal) {
|
|
226
232
|
if (!finals.has(i)) {
|
|
227
233
|
finals.add(i);
|
|
234
|
+
startedAt ??= starts.get(i);
|
|
228
235
|
final.push(text);
|
|
229
236
|
}
|
|
230
237
|
} else interim.push(text);
|
|
231
238
|
}
|
|
232
|
-
notify(session.onResult, { interim: interim.join(' '), final: final.join(' ') });
|
|
239
|
+
notify(session.onResult, { interim: interim.join(' '), final: final.join(' '), startedAt });
|
|
233
240
|
};
|
|
234
241
|
recognition.onerror = (event) => {
|
|
235
242
|
if (!valid()) return;
|
|
@@ -109,6 +109,7 @@ export class WhisperHttpRecognitionEngine {
|
|
|
109
109
|
chunks: [],
|
|
110
110
|
samples: 0,
|
|
111
111
|
voiced: false,
|
|
112
|
+
startedAt: null,
|
|
112
113
|
silence: 0,
|
|
113
114
|
transcriptionQueue: [],
|
|
114
115
|
activeRequest: null,
|
|
@@ -131,6 +132,7 @@ export class WhisperHttpRecognitionEngine {
|
|
|
131
132
|
const enqueue = () => {
|
|
132
133
|
if (!session.voiced || session.samples < context.sampleRate * 0.25) {
|
|
133
134
|
session.chunks = [];
|
|
135
|
+
session.startedAt = null;
|
|
134
136
|
session.samples = 0;
|
|
135
137
|
session.voiced = false;
|
|
136
138
|
session.silence = 0;
|
|
@@ -142,11 +144,13 @@ export class WhisperHttpRecognitionEngine {
|
|
|
142
144
|
samples.set(chunk, at);
|
|
143
145
|
at += chunk.length;
|
|
144
146
|
}
|
|
147
|
+
const startedAt = session.startedAt;
|
|
148
|
+
session.startedAt = null;
|
|
145
149
|
session.chunks = [];
|
|
146
150
|
session.samples = 0;
|
|
147
151
|
session.voiced = false;
|
|
148
152
|
session.silence = 0;
|
|
149
|
-
session.transcriptionQueue.push(samples);
|
|
153
|
+
session.transcriptionQueue.push({ samples, startedAt });
|
|
150
154
|
notifyProcessing();
|
|
151
155
|
void drain();
|
|
152
156
|
};
|
|
@@ -155,7 +159,7 @@ export class WhisperHttpRecognitionEngine {
|
|
|
155
159
|
session.draining = true;
|
|
156
160
|
try {
|
|
157
161
|
while (valid() && session.transcriptionQueue.length) {
|
|
158
|
-
const samples = session.transcriptionQueue.shift();
|
|
162
|
+
const { samples, startedAt } = session.transcriptionQueue.shift();
|
|
159
163
|
const request = new AbortController();
|
|
160
164
|
session.activeRequest = request;
|
|
161
165
|
notifyProcessing();
|
|
@@ -175,7 +179,8 @@ export class WhisperHttpRecognitionEngine {
|
|
|
175
179
|
const json = await response.json();
|
|
176
180
|
if (!response.ok || !json?.ok)
|
|
177
181
|
throw new Error(json?.error?.message || 'HTTP transcription failed.');
|
|
178
|
-
if (valid() && json.value.text)
|
|
182
|
+
if (valid() && json.value.text)
|
|
183
|
+
onResult?.({ final: json.value.text, interim: '', startedAt });
|
|
179
184
|
} catch (error) {
|
|
180
185
|
if (error.name !== 'AbortError' && valid()) onError?.(error);
|
|
181
186
|
} finally {
|
|
@@ -193,6 +198,8 @@ export class WhisperHttpRecognitionEngine {
|
|
|
193
198
|
const data = new Float32Array(event.inputBuffer.getChannelData(0)),
|
|
194
199
|
rms = Math.sqrt(data.reduce((sum, x) => sum + x * x, 0) / data.length);
|
|
195
200
|
if (rms > 0.012) {
|
|
201
|
+
if (!session.voiced)
|
|
202
|
+
session.startedAt = Date.now() - Math.round((data.length / context.sampleRate) * 1000);
|
|
196
203
|
session.voiced = true;
|
|
197
204
|
session.silence = 0;
|
|
198
205
|
onActivity?.(true);
|