dsh-live-voice 0.3.2 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +48 -14
  2. package/lib/client.js +6474 -3396
  3. package/lib/server.js +29 -6
  4. package/package.json +9 -30
  5. package/src/app/AGENTS.md +1 -1
  6. package/src/app/ARCHITECTURE.md +1 -1
  7. package/src/app/client/apply.tsx +242 -35
  8. package/src/app/client/composerSelection.ts +92 -0
  9. package/src/app/client/i18n/catalogs/base.ts +35 -0
  10. package/src/app/client/i18n/catalogs/en.ts +43 -3
  11. package/src/app/client/i18n/catalogs/es.ts +51 -11
  12. package/src/app/client/i18n/catalogs/fr.ts +52 -11
  13. package/src/app/client/i18n/catalogs/hi.ts +50 -11
  14. package/src/app/client/i18n/catalogs/pt-BR.ts +52 -11
  15. package/src/app/client/i18n/catalogs/zh.ts +50 -11
  16. package/src/modules/AGENTS.md +1 -1
  17. package/src/modules/conversation/ARCHITECTURE.md +6 -2
  18. package/src/modules/conversation/components/AutoPlaybackToggle.tsx +31 -0
  19. package/src/modules/conversation/components/ConversationControls.tsx +23 -8
  20. package/src/modules/conversation/components/ConversationStatusBar.tsx +91 -93
  21. package/src/modules/conversation/components/DeliveryModeButton.tsx +3 -3
  22. package/src/modules/conversation/components/MeetingControls.tsx +40 -0
  23. package/src/modules/conversation/components/MicrophoneButton.tsx +15 -2
  24. package/src/modules/conversation/components/PlaybackControls.tsx +20 -5
  25. package/src/modules/conversation/components/RecognitionBar.tsx +14 -0
  26. package/src/modules/conversation/components/ScrollingSpeechCaption.tsx +100 -0
  27. package/src/modules/conversation/components/SpeechStatusBar.tsx +99 -0
  28. package/src/modules/conversation/components/Waveform.tsx +8 -3
  29. package/src/modules/conversation/components/conversationStatus.ts +1 -3
  30. package/src/modules/conversation/components/createConversationComponents.tsx +1 -1
  31. package/src/modules/conversation/components/index.ts +1 -0
  32. package/src/modules/conversation/components/speechCaption.ts +27 -0
  33. package/src/modules/conversation/models/chat.ts +15 -2
  34. package/src/modules/conversation/models/meeting.ts +256 -0
  35. package/src/modules/conversation/models/meetingTranscript.ts +63 -0
  36. package/src/modules/core/ARCHITECTURE.md +1 -1
  37. package/src/modules/core/coordinator.ts +353 -81
  38. package/src/modules/core/developerExtension.ts +26 -0
  39. package/src/modules/core/diagnostics.ts +153 -0
  40. package/src/modules/core/filters.ts +20 -15
  41. package/src/modules/core/markdownSpeech.ts +154 -0
  42. package/src/modules/core/settings.ts +17 -13
  43. package/src/modules/core/sharedAudio.ts +88 -0
  44. package/src/modules/core/speechDefaults.ts +21 -0
  45. package/src/modules/recognition/engines/browser/BrowserRecognitionEngine.ts +8 -1
  46. package/src/modules/recognition/engines/whisper/WhisperRecognitionEngine.ts +10 -3
  47. package/src/modules/settings/components/LiveVoiceSettings.tsx +17 -3
  48. package/src/modules/settings/components/SettingsHeader.tsx +1 -1
  49. package/src/modules/settings/models/settingsStorage.ts +11 -3
  50. package/src/modules/settings/sections/conversation/ConversationDelaySettings.tsx +8 -3
  51. package/src/modules/settings/sections/conversation/DeliverySettings.tsx +25 -2
  52. package/src/modules/settings/sections/recognition/RecognitionEngineSettings.tsx +2 -1
  53. package/src/modules/settings/sections/speak/SpeechEngineSettings.tsx +12 -1
  54. package/src/modules/speak/engines/audio/HostAudioEngine.ts +19 -2
  55. package/src/modules/speak/engines/browser/BrowserSpeakingEngine.ts +33 -2
  56. package/src/shared/design-system/buttons/ToggleButton.tsx +8 -1
  57. package/src/shared/design-system/icons/icons.ts +6 -0
  58. package/src/styles/index.ts +13 -3
  59. package/CHANGELOG.md +0 -240
  60. package/DEVELOPMENT.md +0 -83
  61. package/PLAN.md +0 -296
  62. package/docs/ARCHITECTURE.md +0 -33
  63. package/docs/CHOOSING-AN-ENGINE.md +0 -120
  64. package/docs/CONFIGURATION.md +0 -163
  65. package/docs/REVIEW.md +0 -35
  66. package/docs/VOICE-LIFECYCLE.md +0 -63
  67. package/scripts/build.ts +0 -62
  68. package/scripts/check-dist.mjs +0 -38
  69. package/scripts/preview-ui.ts +0 -31
  70. package/scripts/probe-browser.ts +0 -46
@@ -0,0 +1,26 @@
1
+ import type { ComponentType } from 'react';
2
+ export const DEVELOPER_KEY = Symbol.for('dsh-live-voice.developer.v1');
3
+ export const DEVELOPER_CHANGED = 'dsh-live-voice:developer-changed';
4
+ export interface DeveloperExtension {
5
+ version: 1;
6
+ label(): string;
7
+ component: ComponentType;
8
+ }
9
+ export function readDeveloperExtension(target: any = globalThis.window): DeveloperExtension | null {
10
+ const extension = target?.[DEVELOPER_KEY];
11
+ return extension?.version === 1 ? extension : null;
12
+ }
13
+ export function subscribeDeveloperExtension(listener: () => void, target: any = globalThis.window) {
14
+ target?.addEventListener(DEVELOPER_CHANGED, listener);
15
+ return () => target?.removeEventListener(DEVELOPER_CHANGED, listener);
16
+ }
17
+ export function publishDeveloperExtension(target: any, extension: DeveloperExtension) {
18
+ const ChangedEvent = target.Event ?? Event;
19
+ target[DEVELOPER_KEY] = extension;
20
+ target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
21
+ return () => {
22
+ if (target[DEVELOPER_KEY] !== extension) return;
23
+ delete target[DEVELOPER_KEY];
24
+ target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
25
+ };
26
+ }
@@ -0,0 +1,153 @@
1
+ /** Versioned read-only boundary; debugger UI is never imported by the runtime. */
2
+ export const DIAGNOSTIC_KEY = Symbol.for('dsh-live-voice.diagnostics.v1');
3
+ export const DIAGNOSTIC_CHANGED = 'dsh-live-voice:diagnostics-changed';
4
+ export type DiagnosticValue =
5
+ null | boolean | number | string | DiagnosticValue[] | { [key: string]: DiagnosticValue };
6
+ export interface DiagnosticSource {
7
+ version: 1;
8
+ read(options?: { includeContent?: boolean }): DiagnosticValue;
9
+ subscribe(listener: () => void): () => void;
10
+ }
11
+ export function publishDiagnosticSource(target: any, source: DiagnosticSource) {
12
+ target[DIAGNOSTIC_KEY] = source;
13
+ const ChangedEvent = target.Event ?? Event;
14
+ target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
15
+ return () => {
16
+ if (target[DIAGNOSTIC_KEY] !== source) return;
17
+ delete target[DIAGNOSTIC_KEY];
18
+ target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
19
+ };
20
+ }
21
+ const number = (value: unknown) =>
22
+ typeof value === 'number' && Number.isFinite(value) ? value : null;
23
+ /** Explicit projection excludes audio, functions, resources and free-form errors. */
24
+ export function readCoordinatorDiagnostics(
25
+ controller: any,
26
+ includeContent = false,
27
+ ): DiagnosticValue {
28
+ const state = controller.getSnapshot();
29
+ const recognition = controller.recognition?.session;
30
+ const settings: Record<string, DiagnosticValue> = {};
31
+ const publicSettings = new Set([
32
+ 'engine',
33
+ 'recognitionEngine',
34
+ 'voiceDetectionPreset',
35
+ 'sendingMode',
36
+ 'dshBusyEnterBehavior',
37
+ 'mode',
38
+ 'lang',
39
+ 'recognitionLang',
40
+ ]);
41
+ for (const [key, value] of Object.entries(state.settings ?? {})) {
42
+ settings[key] =
43
+ typeof value === 'string'
44
+ ? includeContent || publicSettings.has(key)
45
+ ? value
46
+ : { characters: value.length }
47
+ : typeof value === 'boolean' || typeof value === 'number'
48
+ ? value
49
+ : null;
50
+ }
51
+ const item = (entry: any): DiagnosticValue => ({
52
+ messageId: entry.id == null ? null : String(entry.id),
53
+ manual: entry.manual === true,
54
+ characters: entry.text?.length ?? 0,
55
+ ...(includeContent ? { text: String(entry.text ?? '') } : {}),
56
+ preparing: Boolean(entry.preparing),
57
+ prepared: Boolean(entry.prepared),
58
+ failed: Boolean(entry.prepareError),
59
+ });
60
+ return {
61
+ settings,
62
+ lifecycle: {
63
+ disposed: Boolean(controller.disposed),
64
+ conversation: Boolean(state.conversation),
65
+ },
66
+ composer: {
67
+ characters: controller.composer?.getDraft?.()?.length ?? 0,
68
+ answeringQuestion: Boolean(state.answeringQuestion),
69
+ ...(includeContent ? { draft: controller.composer?.getDraft?.() ?? '' } : {}),
70
+ },
71
+ capture: {
72
+ starting: Boolean(state.starting),
73
+ listening: Boolean(state.listening),
74
+ muted: Boolean(state.muted),
75
+ generation: number(controller.epoch),
76
+ level: number(controller.meter?.level?.()),
77
+ releaseFailed: Boolean(controller.inputReleaseError),
78
+ },
79
+ vad: {
80
+ source: state.settings?.recognitionEngine === 'browser' ? 'native' : 'plugin',
81
+ activity: Boolean(state.recognizing),
82
+ threshold: recognition?.chunks ? 0.012 : null,
83
+ silenceMs:
84
+ recognition?.chunks && controller.meter?.context?.sampleRate
85
+ ? (recognition.silence / controller.meter.context.sampleRate) * 1000
86
+ : null,
87
+ },
88
+ chunker: {
89
+ available: Boolean(recognition?.chunks),
90
+ bufferedSamples: number(recognition?.samples),
91
+ bufferedChunks: recognition?.chunks?.length ?? null,
92
+ containsSpeech: recognition?.voiced ?? null,
93
+ },
94
+ recognition: {
95
+ pendingTranscriptions: number(state.pendingTranscriptions),
96
+ activeRequest: Boolean(recognition?.activeRequest),
97
+ queue: (recognition?.transcriptionQueue ?? []).map((chunk: any) => ({
98
+ samples: (chunk.samples ?? chunk).length,
99
+ durationMs: controller.meter?.context?.sampleRate
100
+ ? ((chunk.samples ?? chunk).length / controller.meter.context.sampleRate) * 1000
101
+ : null,
102
+ })),
103
+ },
104
+ delivery: {
105
+ autoSendAt: number(state.autoSendAt),
106
+ countdownActive: controller.autoSendTimer != null,
107
+ holdToTalkRelease: Boolean(controller.holdToTalkRelease),
108
+ },
109
+ speech: {
110
+ configuredEngine: state.settings?.engine ?? null,
111
+ activeEngine: state.speechEngine ?? null,
112
+ activeEndpoint: controller.engines?.[state.speechEngine]?.endpoint ?? null,
113
+ speaking: Boolean(state.speaking),
114
+ paused: Boolean(state.paused),
115
+ activeMessageId: state.activeMessageId == null ? null : String(state.activeMessageId),
116
+ generation: number(controller.speechEpoch),
117
+ controlGeneration: number(controller.controlEpoch),
118
+ remaining: number(state.speechSegmentsRemaining),
119
+ notBefore: number(controller.assistantSpeechNotBefore),
120
+ waitingForActivity: controller.assistantSpeechNotBefore === Infinity,
121
+ stopFailed: Boolean(controller.speechStopError),
122
+ queue: (controller.queue ?? []).map(item),
123
+ activeItem: controller.activeSpeechItem ? item(controller.activeSpeechItem) : null,
124
+ prefetchCount: controller.prefetchItems?.size ?? 0,
125
+ consumed: Object.fromEntries(controller.consumed ?? []),
126
+ unfinished: [...(controller.unfinished ?? [])].map(String),
127
+ suppressed: [...(controller.suppressed ?? [])].map(String),
128
+ },
129
+ interruption: {
130
+ transcriptConfirmed: Boolean(controller.interruptionTranscriptConfirmed),
131
+ pausedSpeech: Boolean(controller.interruptionPausedSpeech),
132
+ timerActive: controller.interruptionTimer != null,
133
+ },
134
+ capabilities: Object.fromEntries(
135
+ Object.entries(state.capabilities ?? {}).map(([key, cap]: [string, any]) => [
136
+ key,
137
+ cap
138
+ ? {
139
+ supported: cap.supported === true,
140
+ local: cap.local ?? null,
141
+ pause: cap.pause ?? null,
142
+ resume: cap.resume ?? null,
143
+ hasReason: Boolean(cap.reason),
144
+ }
145
+ : null,
146
+ ]),
147
+ ),
148
+ error: {
149
+ present: Boolean(state.error),
150
+ ...(includeContent ? { message: state.error ?? null } : {}),
151
+ },
152
+ };
153
+ }
@@ -1,4 +1,6 @@
1
1
  // @ts-nocheck
2
+ import { markdownSpeech } from './markdownSpeech.js';
3
+ export { hasIncompleteSpeechMarkdown } from './markdownSpeech.js';
2
4
  const words = (text) =>
3
5
  String(text || '')
4
6
  .trim()
@@ -41,23 +43,26 @@ export function splitSpeechOutput(text, options = {}) {
41
43
  .filter(Boolean);
42
44
  }
43
45
 
46
+ /** Split only the initial item at sentence punctuation or colons, never commas. */
47
+ export function splitInitialSpeechItem(text) {
48
+ const result = [];
49
+ let start = 0;
50
+ // Require whitespace/end after punctuation to preserve decimals, URLs and file names.
51
+ for (const match of text.matchAll(/[.!?:]+(?:["”’)]*)?(?=\s|$)/gu)) {
52
+ const end = match.index + match[0].length;
53
+ const part = text.slice(start, end).trim();
54
+ if (part) result.push(part);
55
+ start = end;
56
+ }
57
+ const tail = text.slice(start).trim();
58
+ if (tail) result.push(tail);
59
+ return result.length ? result : [text];
60
+ }
61
+
44
62
  export function hasUnclosedCodeFence(text) {
45
63
  return (String(text || '').match(/```/g) || []).length % 2 === 1;
46
64
  }
47
65
 
48
- export function filterSpeechOutput(
49
- text,
50
- {
51
- filterCodeBlocks = true,
52
- codeBlockMaxLines = 5,
53
- codeBlockNotice = 'Look the code on out conversation',
54
- } = {},
55
- ) {
56
- const source = String(text || '');
57
- if (!filterCodeBlocks) return source;
58
- return source.replace(/```[^\n]*\n?([\s\S]*?)```/g, (block, body) => {
59
- const normalized = body.replace(/\n$/, '');
60
- const lineCount = normalized ? normalized.split(/\r?\n/).length : 0;
61
- return lineCount <= codeBlockMaxLines ? normalized : codeBlockNotice;
62
- });
66
+ export function filterSpeechOutput(text, options = {}) {
67
+ return markdownSpeech(text, options);
63
68
  }
@@ -0,0 +1,154 @@
1
+ // @ts-nocheck
2
+ import { marked } from 'marked';
3
+ import { resolveCodeNotice } from './speechDefaults.js';
4
+
5
+ // Spoken phrases follow the TTS language, independently of the interface locale.
6
+ const phrases = {
7
+ en: [
8
+ 'Link to',
9
+ 'Image:',
10
+ 'Table with columns:',
11
+ 'Checked:',
12
+ 'Unchecked:',
13
+ 'line',
14
+ 'lines',
15
+ 'to',
16
+ 'identifier',
17
+ ],
18
+ pt: [
19
+ 'Link para',
20
+ 'Imagem:',
21
+ 'Tabela com colunas:',
22
+ 'Marcado:',
23
+ 'Desmarcado:',
24
+ 'linha',
25
+ 'linhas',
26
+ 'a',
27
+ 'identificador',
28
+ ],
29
+ es: [
30
+ 'Enlace a',
31
+ 'Imagen:',
32
+ 'Tabla con columnas:',
33
+ 'Marcado:',
34
+ 'Desmarcado:',
35
+ 'línea',
36
+ 'líneas',
37
+ 'a',
38
+ 'identificador',
39
+ ],
40
+ fr: [
41
+ 'Lien vers',
42
+ 'Image :',
43
+ 'Tableau avec les colonnes :',
44
+ 'Coché :',
45
+ 'Décoché :',
46
+ 'ligne',
47
+ 'lignes',
48
+ 'à',
49
+ 'identifiant',
50
+ ],
51
+ hi: [
52
+ 'लिंक',
53
+ 'चित्र:',
54
+ 'तालिका के स्तंभ:',
55
+ 'चिह्नित:',
56
+ 'अचिह्नित:',
57
+ 'पंक्ति',
58
+ 'पंक्तियाँ',
59
+ 'से',
60
+ 'पहचानकर्ता',
61
+ ],
62
+ zh: ['链接到', '图片:', '表格列:', '已勾选:', '未勾选:', '行', '行', '至', '标识符'],
63
+ };
64
+ export function markdownSpeech(source, options = {}) {
65
+ const p = phrases[String(options.lang || 'en').split('-')[0]] || phrases.en;
66
+ const host = (url) => {
67
+ try {
68
+ return new URL(url).hostname;
69
+ } catch {
70
+ return '';
71
+ }
72
+ };
73
+ const plain = (text) =>
74
+ String(text)
75
+ .replace(/https?:\/\/[^\s<>]+/gu, (url) => p[0] + ' ' + host(url))
76
+ .replace(
77
+ /(?:[A-Za-z]:[\\/]|\.{0,2}\/)?(?:[\w.@~-]+[\\/])+([\w.@~-]+\.[\w]+)((?::\d+(?:-\d+)?)|(?:#L\d+(?:-L?\d+)?))?/gu,
78
+ (_, name, loc = '') => name + loc,
79
+ )
80
+ .replace(
81
+ /([\w.@~-]+\.[\w]+)(?::(\d+)(?:-(\d+))?|#L(\d+)(?:-L?(\d+))?)/gu,
82
+ (_, name, a, b, c, d) =>
83
+ name +
84
+ ', ' +
85
+ (b || d ? p[6] : p[5]) +
86
+ ' ' +
87
+ (a || c) +
88
+ (b || d ? ' ' + p[7] + ' ' + (b || d) : ''),
89
+ )
90
+ .replace(
91
+ /\b(?:[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}|[a-f\d]{24,})\b/giu,
92
+ p[8],
93
+ );
94
+ const inline = (tokens = []) =>
95
+ tokens
96
+ .map((t) => {
97
+ if (t.type === 'link') {
98
+ const name = t.text === t.href ? host(t.href) : inline(t.tokens);
99
+ return p[0] + ' ' + (name || host(t.href) || plain(t.href));
100
+ }
101
+ if (t.type === 'image') return t.text ? p[1] + ' ' + plain(t.text) : '';
102
+ if (t.type === 'html') return '';
103
+ if (t.type === 'br') return '\n';
104
+ if (t.tokens) return inline(t.tokens);
105
+ return plain(t.text ?? t.raw ?? '');
106
+ })
107
+ .join('');
108
+ const blocks = (tokens) =>
109
+ tokens
110
+ .map((t) => {
111
+ if (t.type === 'space') return t.raw;
112
+ if (t.type === 'hr' || t.type === 'def' || t.type === 'html') return '';
113
+ if (t.type === 'code')
114
+ return options.filterCodeBlocks !== false &&
115
+ t.text.split(/\r?\n/).length > (options.codeBlockMaxLines ?? 5)
116
+ ? resolveCodeNotice(options.codeBlockNotice, options.lang)
117
+ : t.text;
118
+ if (t.type === 'blockquote') return blocks(t.tokens);
119
+ if (t.type === 'list')
120
+ return t.items
121
+ .map(
122
+ (item, i) =>
123
+ (item.task
124
+ ? (item.checked ? p[3] : p[4]) + ' '
125
+ : t.ordered
126
+ ? Number(t.start) + i + '. '
127
+ : '') + blocks(item.tokens).trim(),
128
+ )
129
+ .join('\n');
130
+ if (t.type === 'table')
131
+ return (
132
+ p[2] +
133
+ ' ' +
134
+ t.header.map((c) => inline(c.tokens)).join(', ') +
135
+ '\n' +
136
+ t.rows.map((row) => row.map((c) => inline(c.tokens)).join(', ')).join('\n')
137
+ );
138
+ return t.tokens ? inline(t.tokens) : plain(t.text || '');
139
+ })
140
+ .reduce(
141
+ (result, value) =>
142
+ !value
143
+ ? result
144
+ : result +
145
+ (result && !result.endsWith('\n') && !value.startsWith('\n') ? '\n' : '') +
146
+ value,
147
+ '',
148
+ );
149
+ return blocks(marked.lexer(String(source || ''), { gfm: true })).trim();
150
+ }
151
+ export function hasIncompleteSpeechMarkdown(source) {
152
+ if (/^[ \t]*\|/mu.test(source) || /\]\[[^\]]*\]/u.test(source)) return true;
153
+ return /!?\[[^\]]*$|!?\[[^\]]*\]\([^)]*$|!?\[[^\]]*\]\[[^\]]*\]/u.test(source);
154
+ }
@@ -1,4 +1,5 @@
1
1
  // @ts-nocheck
2
+ import { resolveCodeNotice } from './speechDefaults.js';
2
3
  export const voiceDetectionPresets = Object.freeze({
3
4
  short: Object.freeze({
4
5
  silenceMs: 500,
@@ -34,6 +35,8 @@ export const defaultAgentVoiceContext = `Live Voice output is active. Your entir
34
35
  Be concise and conversational. Lead with the answer or next action. Avoid unnecessary repetition, long preambles, dense lists, raw code, paths, and verbose status narration.
35
36
  Do not narrate routine tool activity by default. If the user explicitly asks you to keep them informed while working, provide brief spoken progress updates only at meaningful milestones.`;
36
37
 
38
+ export const autoSendDelayOptions = Object.freeze([0.6, 0.8, 1, 2, 3, 4, 5, 6]);
39
+
37
40
  export const defaultSettings = Object.freeze({
38
41
  engine: 'browser',
39
42
  recognitionEngine: 'browser',
@@ -43,14 +46,15 @@ export const defaultSettings = Object.freeze({
43
46
  voiceDetectionCustomSilenceMs: 1000,
44
47
  recognitionMaxUtteranceSeconds: 60,
45
48
  microphoneEnabled: true,
46
- holdToTalkEnabled: true,
49
+ holdToTalkEnabled: false,
47
50
  announceAssistantMessages: true,
48
51
  agentVoiceContextEnabled: true,
49
52
  agentVoiceContext: defaultAgentVoiceContext,
50
53
  interruptSpeechOnUserMessage: false,
51
54
  sendingMode: 'manual',
55
+ dshBusyEnterBehavior: 'queue',
52
56
  autoSendDelaySeconds: 4,
53
- assistantSpeechDelaySeconds: 3,
57
+ assistantSpeechDelaySeconds: 0,
54
58
  mode: 'speaker',
55
59
  lang: 'pt-BR',
56
60
  recognitionLang: 'pt-BR',
@@ -69,7 +73,7 @@ export const defaultSettings = Object.freeze({
69
73
  voiceCommandClear: 'clear all, clear message',
70
74
  outputCodeFilterEnabled: true,
71
75
  outputCodeMaxLines: 5,
72
- outputCodeNotice: 'Look the code on out conversation',
76
+ outputCodeNotice: resolveCodeNotice(undefined, 'pt-BR'),
73
77
  rate: 1,
74
78
  segmentGapMs: 400,
75
79
  });
@@ -153,22 +157,22 @@ export function normalizeSettings(value) {
153
157
  typeof source.interruptSpeechOnUserMessage === 'boolean'
154
158
  ? source.interruptSpeechOnUserMessage
155
159
  : defaultSettings.interruptSpeechOnUserMessage,
160
+ dshBusyEnterBehavior: ['queue', 'steer'].includes(source.dshBusyEnterBehavior)
161
+ ? source.dshBusyEnterBehavior
162
+ : defaultSettings.dshBusyEnterBehavior,
156
163
  sendingMode:
157
164
  source.sendingMode === 'automatic'
158
165
  ? 'queue'
159
166
  : ['manual', 'queue', 'steer'].includes(source.sendingMode)
160
167
  ? source.sendingMode
161
168
  : defaultSettings.sendingMode,
162
- autoSendDelaySeconds:
163
- Number.isInteger(source.autoSendDelaySeconds) &&
164
- source.autoSendDelaySeconds >= 2 &&
165
- source.autoSendDelaySeconds <= 10
166
- ? source.autoSendDelaySeconds
167
- : defaultSettings.autoSendDelaySeconds,
169
+ autoSendDelaySeconds: autoSendDelayOptions.includes(source.autoSendDelaySeconds)
170
+ ? source.autoSendDelaySeconds
171
+ : defaultSettings.autoSendDelaySeconds,
168
172
  assistantSpeechDelaySeconds:
169
173
  Number.isInteger(source.assistantSpeechDelaySeconds) &&
170
- source.assistantSpeechDelaySeconds >= 1 &&
171
- source.assistantSpeechDelaySeconds <= 10
174
+ source.assistantSpeechDelaySeconds >= 0 &&
175
+ source.assistantSpeechDelaySeconds <= 4
172
176
  ? source.assistantSpeechDelaySeconds
173
177
  : defaultSettings.assistantSpeechDelaySeconds,
174
178
  mode: ['speaker', 'headphones'].includes(source.mode) ? source.mode : defaultSettings.mode,
@@ -257,8 +261,8 @@ export function normalizeSettings(value) {
257
261
  source.outputCodeNotice.trim() &&
258
262
  source.outputCodeNotice.length <= 300 &&
259
263
  !source.outputCodeNotice.includes('\0')
260
- ? source.outputCodeNotice.trim()
261
- : defaultSettings.outputCodeNotice,
264
+ ? resolveCodeNotice(source.outputCodeNotice, source.lang || defaultSettings.lang)
265
+ : resolveCodeNotice(defaultSettings.outputCodeNotice, source.lang || defaultSettings.lang),
262
266
  rate: Number.isFinite(source.rate) && source.rate >= 0.1 && source.rate <= 3 ? source.rate : 1,
263
267
  segmentGapMs:
264
268
  Number.isFinite(source.segmentGapMs) &&
@@ -0,0 +1,88 @@
1
+ // @ts-nocheck
2
+ import { MicrophoneMeter } from './microphone.js';
3
+
4
+ /** Display permission must be requested synchronously from a user gesture. */
5
+ export function requestSharedAudio(globals = globalThis) {
6
+ return globals.navigator.mediaDevices.getDisplayMedia({
7
+ video: true,
8
+ audio: true,
9
+ // Chromium hints exclude this application's own tab; unsupported browsers ignore them.
10
+ selfBrowserSurface: 'exclude',
11
+ restrictOwnAudio: true,
12
+ });
13
+ }
14
+
15
+ export class SharedAudioMeter extends MicrophoneMeter {
16
+ constructor(globals = globalThis) {
17
+ super(globals);
18
+ this.pending = null;
19
+ }
20
+ provide(request) {
21
+ this.pending = Promise.resolve(request);
22
+ this.pending.catch(() => {});
23
+ }
24
+ async capability() {
25
+ return {
26
+ supported:
27
+ typeof this.g.navigator?.mediaDevices?.getDisplayMedia === 'function' &&
28
+ typeof (this.g.AudioContext || this.g.webkitAudioContext) === 'function',
29
+ };
30
+ }
31
+ async stop() {
32
+ const pending = this.pending;
33
+ this.pending = null;
34
+ pending?.then((stream) => stream.getTracks().forEach((track) => track.stop())).catch(() => {});
35
+ return super.stop();
36
+ }
37
+ async start({ signal } = {}) {
38
+ const pending = this.pending;
39
+ this.pending = null;
40
+ await super.stop();
41
+ if (!pending) throw new Error('Shared audio permission was not requested.');
42
+ const job = {
43
+ signal,
44
+ stream: null,
45
+ context: null,
46
+ source: null,
47
+ cancel: () => {
48
+ if (this.current === job) void this.stop();
49
+ },
50
+ cancelled: () => {},
51
+ };
52
+ this.current = job;
53
+ signal?.addEventListener('abort', job.cancel, { once: true });
54
+ try {
55
+ const stream = await pending;
56
+ job.stream = stream;
57
+ if (signal?.aborted || this.current !== job) {
58
+ this._dispose(job);
59
+ return false;
60
+ }
61
+ const tracks = stream.getAudioTracks();
62
+ if (!tracks.length) throw new Error('No shared audio track was returned.');
63
+ const AudioContext = this.g.AudioContext || this.g.webkitAudioContext;
64
+ job.context = new AudioContext();
65
+ // Exclude video and never route captured audio to audible output.
66
+ job.source = job.context.createMediaStreamSource(new this.g.MediaStream(tracks));
67
+ job.analyser = job.context.createAnalyser();
68
+ job.analyser.fftSize = 256;
69
+ job.source.connect(job.analyser);
70
+ job.samples = new Float32Array(job.analyser.fftSize);
71
+ this.stream = stream;
72
+ this.context = job.context;
73
+ this.source = job.source;
74
+ this.analyser = job.analyser;
75
+ this.samples = job.samples;
76
+ await job.context.resume();
77
+ return this.current === job;
78
+ } catch (error) {
79
+ if (this.current === job) {
80
+ this.current = null;
81
+ this._clear();
82
+ }
83
+ this._dispose(job);
84
+ if (signal?.aborted) return false;
85
+ throw error;
86
+ }
87
+ }
88
+ }
@@ -0,0 +1,21 @@
1
+ const notices: Record<string, string> = {
2
+ en: 'You can see the code in our conversation history.',
3
+ pt: 'Você pode ver o código no histórico da nossa conversa.',
4
+ es: 'Puedes ver el código en el historial de nuestra conversación.',
5
+ fr: 'Vous pouvez voir le code dans l’historique de notre conversation.',
6
+ hi: 'आप हमारी बातचीत के इतिहास में कोड देख सकते हैं।',
7
+ zh: '你可以在我们的对话历史中查看代码。',
8
+ };
9
+ const defaults = new Set([
10
+ ...Object.values(notices),
11
+ 'Look the code on out conversation',
12
+ 'You can see the code on our conversation history.',
13
+ 'You can see the code on our conversation history',
14
+ ]);
15
+ /** Default notices follow TTS language; explicit custom notices remain unchanged. */
16
+ export function resolveCodeNotice(value: unknown, lang = 'en') {
17
+ const text = typeof value === 'string' ? value.trim() : '';
18
+ return !text || defaults.has(text)
19
+ ? notices[String(lang).toLowerCase().split('-')[0]] || notices.en
20
+ : text;
21
+ }
@@ -205,10 +205,13 @@ export class BrowserRecognitionEngine {
205
205
  recognition.interimResults = true;
206
206
  session.recognition = recognition;
207
207
  const finals = new Set();
208
+ const starts = new Map();
209
+ let speechStartedAt = null;
208
210
  const valid = () => this.session === session && session.recognition === recognition;
209
211
  recognition.onspeechstart = () => {
210
212
  if (!valid()) return;
211
213
  session.restarts = 0;
214
+ speechStartedAt = Date.now();
212
215
  this._activity(session, true);
213
216
  };
214
217
  recognition.onspeechend = () => {
@@ -216,20 +219,24 @@ export class BrowserRecognitionEngine {
216
219
  };
217
220
  recognition.onresult = (event) => {
218
221
  if (!valid()) return;
222
+ let startedAt = null;
219
223
  const interim = [],
220
224
  final = [];
221
225
  for (let i = 0; i < event.results.length; i++) {
222
226
  const result = event.results[i];
223
227
  const text = result[0]?.transcript ?? '';
228
+ if (!finals.has(i) && !starts.has(i) && text.trim())
229
+ starts.set(i, speechStartedAt ?? Date.now());
224
230
  if (text.trim()) session.restarts = 0;
225
231
  if (result.isFinal) {
226
232
  if (!finals.has(i)) {
227
233
  finals.add(i);
234
+ startedAt ??= starts.get(i);
228
235
  final.push(text);
229
236
  }
230
237
  } else interim.push(text);
231
238
  }
232
- notify(session.onResult, { interim: interim.join(' '), final: final.join(' ') });
239
+ notify(session.onResult, { interim: interim.join(' '), final: final.join(' '), startedAt });
233
240
  };
234
241
  recognition.onerror = (event) => {
235
242
  if (!valid()) return;
@@ -109,6 +109,7 @@ export class WhisperHttpRecognitionEngine {
109
109
  chunks: [],
110
110
  samples: 0,
111
111
  voiced: false,
112
+ startedAt: null,
112
113
  silence: 0,
113
114
  transcriptionQueue: [],
114
115
  activeRequest: null,
@@ -131,6 +132,7 @@ export class WhisperHttpRecognitionEngine {
131
132
  const enqueue = () => {
132
133
  if (!session.voiced || session.samples < context.sampleRate * 0.25) {
133
134
  session.chunks = [];
135
+ session.startedAt = null;
134
136
  session.samples = 0;
135
137
  session.voiced = false;
136
138
  session.silence = 0;
@@ -142,11 +144,13 @@ export class WhisperHttpRecognitionEngine {
142
144
  samples.set(chunk, at);
143
145
  at += chunk.length;
144
146
  }
147
+ const startedAt = session.startedAt;
148
+ session.startedAt = null;
145
149
  session.chunks = [];
146
150
  session.samples = 0;
147
151
  session.voiced = false;
148
152
  session.silence = 0;
149
- session.transcriptionQueue.push(samples);
153
+ session.transcriptionQueue.push({ samples, startedAt });
150
154
  notifyProcessing();
151
155
  void drain();
152
156
  };
@@ -155,7 +159,7 @@ export class WhisperHttpRecognitionEngine {
155
159
  session.draining = true;
156
160
  try {
157
161
  while (valid() && session.transcriptionQueue.length) {
158
- const samples = session.transcriptionQueue.shift();
162
+ const { samples, startedAt } = session.transcriptionQueue.shift();
159
163
  const request = new AbortController();
160
164
  session.activeRequest = request;
161
165
  notifyProcessing();
@@ -175,7 +179,8 @@ export class WhisperHttpRecognitionEngine {
175
179
  const json = await response.json();
176
180
  if (!response.ok || !json?.ok)
177
181
  throw new Error(json?.error?.message || 'HTTP transcription failed.');
178
- if (valid() && json.value.text) onResult?.({ final: json.value.text, interim: '' });
182
+ if (valid() && json.value.text)
183
+ onResult?.({ final: json.value.text, interim: '', startedAt });
179
184
  } catch (error) {
180
185
  if (error.name !== 'AbortError' && valid()) onError?.(error);
181
186
  } finally {
@@ -193,6 +198,8 @@ export class WhisperHttpRecognitionEngine {
193
198
  const data = new Float32Array(event.inputBuffer.getChannelData(0)),
194
199
  rms = Math.sqrt(data.reduce((sum, x) => sum + x * x, 0) / data.length);
195
200
  if (rms > 0.012) {
201
+ if (!session.voiced)
202
+ session.startedAt = Date.now() - Math.round((data.length / context.sampleRate) * 1000);
196
203
  session.voiced = true;
197
204
  session.silence = 0;
198
205
  onActivity?.(true);