@phenx-inc/ctlsurf 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/headless/index.mjs +169 -10
- package/out/headless/index.mjs.map +2 -2
- package/out/main/index.js +175 -9
- package/out/preload/index.js +10 -0
- package/out/renderer/assets/{cssMode-eTXVdAkZ.js → cssMode-BdYp0fp9.js} +3 -3
- package/out/renderer/assets/{freemarker2-B5BKaiK4.js → freemarker2-Cya-9zr-.js} +1 -1
- package/out/renderer/assets/{handlebars-BIdLd2wU.js → handlebars-EJs9W5Ub.js} +1 -1
- package/out/renderer/assets/{html-BXL4cnLS.js → html-Ex4qy2lh.js} +1 -1
- package/out/renderer/assets/{htmlMode-46N3XG2c.js → htmlMode-CCm0aJ6C.js} +3 -3
- package/out/renderer/assets/{index-dRvutfbl.js → index-Cj7K9-Ba.js} +823 -137
- package/out/renderer/assets/{index-Cf-RsxoC.css → index-nvsNYP5c.css} +229 -0
- package/out/renderer/assets/{javascript-n_iZZzDX.js → javascript-B2niCouI.js} +2 -2
- package/out/renderer/assets/{jsonMode-DXDczSNu.js → jsonMode-BgCUHaFo.js} +3 -3
- package/out/renderer/assets/{liquid-B1QweUh7.js → liquid-DOQmSo4G.js} +1 -1
- package/out/renderer/assets/{lspLanguageFeatures-DqzMqkRk.js → lspLanguageFeatures-8SIcRAMh.js} +1 -1
- package/out/renderer/assets/{mdx-BCv8lm5e.js → mdx-DuKdGNKE.js} +1 -1
- package/out/renderer/assets/{python-BLNzYwDv.js → python-BUa4XFG-.js} +1 -1
- package/out/renderer/assets/{razor-CvAww8bG.js → razor-BUw8jho0.js} +1 -1
- package/out/renderer/assets/{transformers.web-DtSCnG36.js → transformers.web-_PuZoIho.js} +824 -19
- package/out/renderer/assets/{tsMode-C7m6Kr5E.js → tsMode-8FLRnobw.js} +1 -1
- package/out/renderer/assets/{typescript-DhPw4VVg.js → typescript-rdwAjCi4.js} +1 -1
- package/out/renderer/assets/{xml-B0WLFJ2U.js → xml-HOzT59Qg.js} +1 -1
- package/out/renderer/assets/{yaml-BWyn9Wd7.js → yaml-B81JfLsW.js} +1 -1
- package/out/renderer/index.html +64 -2
- package/out/repro-toggle.mjs +305 -0
- package/package.json +1 -1
- package/src/main/index.ts +9 -0
- package/src/main/orchestrator.ts +51 -10
- package/src/main/transcripts.ts +134 -0
- package/src/preload/index.ts +14 -0
- package/src/renderer/App.tsx +30 -11
- package/src/renderer/components/FloatingMic.tsx +48 -0
- package/src/renderer/components/SpeakControls.tsx +235 -0
- package/src/renderer/components/VoiceInput.tsx +183 -140
- package/src/renderer/index.html +62 -0
- package/src/renderer/lib/localWhisper.ts +94 -6
- package/src/renderer/lib/speech.ts +317 -0
- package/src/renderer/styles.css +229 -0
|
@@ -1,70 +1,26 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef, useState } from 'react'
|
|
2
2
|
import { transcribeBlob, type ModelProgress } from '../lib/localWhisper'
|
|
3
3
|
|
|
4
|
-
// ───
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
|
|
10
|
-
interface SpeechRecognitionResult {
|
|
11
|
-
isFinal: boolean
|
|
12
|
-
0: { transcript: string }
|
|
13
|
-
}
|
|
14
|
-
interface SpeechRecognitionEvent {
|
|
15
|
-
resultIndex: number
|
|
16
|
-
results: { length: number;[index: number]: SpeechRecognitionResult }
|
|
17
|
-
}
|
|
18
|
-
interface SpeechRecognitionErrorEvent { error: string }
|
|
19
|
-
interface SpeechRecognitionLike {
|
|
20
|
-
lang: string
|
|
21
|
-
continuous: boolean
|
|
22
|
-
interimResults: boolean
|
|
23
|
-
start: () => void
|
|
24
|
-
stop: () => void
|
|
25
|
-
abort: () => void
|
|
26
|
-
onresult: ((e: SpeechRecognitionEvent) => void) | null
|
|
27
|
-
onerror: ((e: SpeechRecognitionErrorEvent) => void) | null
|
|
28
|
-
onend: (() => void) | null
|
|
29
|
-
}
|
|
30
|
-
type SpeechRecognitionCtor = new () => SpeechRecognitionLike
|
|
31
|
-
|
|
32
|
-
function getRecognitionCtor(): SpeechRecognitionCtor | null {
|
|
33
|
-
const w = window as unknown as {
|
|
34
|
-
SpeechRecognition?: SpeechRecognitionCtor
|
|
35
|
-
webkitSpeechRecognition?: SpeechRecognitionCtor
|
|
36
|
-
}
|
|
37
|
-
return w.SpeechRecognition || w.webkitSpeechRecognition || null
|
|
38
|
-
}
|
|
4
|
+
// ─── Capabilities ────────────────────────────────────
|
|
5
|
+
// Speech is transcribed entirely on-device via a local Whisper model (see
|
|
6
|
+
// ../lib/localWhisper). The browser's Web Speech API is intentionally not used:
|
|
7
|
+
// it's a cloud service whose Google backend is unreachable inside Electron
|
|
8
|
+
// (Chromium ships without the API key), so it never works in this app.
|
|
39
9
|
|
|
40
|
-
// ─── Capabilities & engine selection ─────────────────
|
|
41
|
-
|
|
42
|
-
type Engine = 'web-speech' | 'local'
|
|
43
10
|
type Phase = 'idle' | 'listening' | 'transcribing'
|
|
44
11
|
|
|
45
|
-
|
|
12
|
+
// Persisted audioinput deviceId; null/absent = OS default.
|
|
13
|
+
const DEVICE_KEY = 'ctlsurf.voiceDeviceId'
|
|
14
|
+
|
|
15
|
+
function loadDeviceId(): string | null {
|
|
16
|
+
try { return localStorage.getItem(DEVICE_KEY) || null } catch { return null }
|
|
17
|
+
}
|
|
46
18
|
|
|
47
|
-
const WEB_SPEECH_SUPPORTED = getRecognitionCtor() !== null
|
|
48
19
|
const LOCAL_SUPPORTED =
|
|
49
20
|
typeof navigator !== 'undefined' &&
|
|
50
21
|
!!navigator.mediaDevices?.getUserMedia &&
|
|
51
22
|
typeof MediaRecorder !== 'undefined' &&
|
|
52
23
|
typeof OfflineAudioContext !== 'undefined'
|
|
53
|
-
const ANY_SUPPORTED = WEB_SPEECH_SUPPORTED || LOCAL_SUPPORTED
|
|
54
|
-
|
|
55
|
-
function loadInitialEngine(): Engine {
|
|
56
|
-
if (!WEB_SPEECH_SUPPORTED && LOCAL_SUPPORTED) return 'local'
|
|
57
|
-
try {
|
|
58
|
-
if (localStorage.getItem(ENGINE_KEY) === 'local' && LOCAL_SUPPORTED) return 'local'
|
|
59
|
-
} catch { /* ignore */ }
|
|
60
|
-
return WEB_SPEECH_SUPPORTED ? 'web-speech' : 'local'
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// Web Speech errors that mean the engine itself is unreachable (vs. a mic
|
|
64
|
-
// permission/hardware problem, which would also break the local fallback).
|
|
65
|
-
function isEngineUnavailable(code: string): boolean {
|
|
66
|
-
return code === 'network' || code === 'service-not-allowed'
|
|
67
|
-
}
|
|
68
24
|
|
|
69
25
|
function describeMicError(err: unknown): string {
|
|
70
26
|
const name = (err as { name?: string })?.name
|
|
@@ -82,26 +38,28 @@ interface VoiceInputProps {
|
|
|
82
38
|
}
|
|
83
39
|
|
|
84
40
|
export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputProps) {
|
|
85
|
-
const [engine, setEngine] = useState<Engine>(loadInitialEngine)
|
|
86
41
|
const [phase, setPhase] = useState<Phase>('idle')
|
|
87
|
-
const [interim, setInterim] = useState('')
|
|
88
42
|
const [modelPct, setModelPct] = useState<number | null>(null)
|
|
43
|
+
// Transcript-so-far streamed from the model during transcription (live preview).
|
|
44
|
+
const [partial, setPartial] = useState('')
|
|
89
45
|
const [error, setError] = useState<string | null>(null)
|
|
90
46
|
const [notice, setNotice] = useState<string | null>(null)
|
|
91
47
|
|
|
92
|
-
//
|
|
93
|
-
const
|
|
94
|
-
const
|
|
95
|
-
|
|
48
|
+
// Mic source selection
|
|
49
|
+
const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
|
|
50
|
+
const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(loadDeviceId)
|
|
51
|
+
const [showDevicePicker, setShowDevicePicker] = useState(false)
|
|
52
|
+
const selectedDeviceIdRef = useRef(selectedDeviceId)
|
|
53
|
+
useEffect(() => { selectedDeviceIdRef.current = selectedDeviceId }, [selectedDeviceId])
|
|
54
|
+
const wrapRef = useRef<HTMLDivElement>(null)
|
|
55
|
+
|
|
56
|
+
// Recording refs
|
|
96
57
|
const streamRef = useRef<MediaStream | null>(null)
|
|
97
58
|
const recorderRef = useRef<MediaRecorder | null>(null)
|
|
98
59
|
const chunksRef = useRef<Blob[]>([])
|
|
99
60
|
// Set true when the user releases before getUserMedia resolves (quick tap).
|
|
100
61
|
const cancelGestureRef = useRef(false)
|
|
101
62
|
|
|
102
|
-
const engineRef = useRef(engine)
|
|
103
|
-
useEffect(() => { engineRef.current = engine }, [engine])
|
|
104
|
-
|
|
105
63
|
const onTranscriptRef = useRef(onTranscript)
|
|
106
64
|
useEffect(() => { onTranscriptRef.current = onTranscript }, [onTranscript])
|
|
107
65
|
|
|
@@ -117,77 +75,94 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
117
75
|
return () => clearTimeout(t)
|
|
118
76
|
}, [notice])
|
|
119
77
|
|
|
120
|
-
const switchToLocal = useCallback((reason: string) => {
|
|
121
|
-
try { localStorage.setItem(ENGINE_KEY, 'local') } catch { /* ignore */ }
|
|
122
|
-
setEngine('local')
|
|
123
|
-
setNotice(reason)
|
|
124
|
-
}, [])
|
|
125
|
-
|
|
126
78
|
const stopStream = useCallback(() => {
|
|
127
79
|
streamRef.current?.getTracks().forEach((t) => t.stop())
|
|
128
80
|
streamRef.current = null
|
|
129
81
|
}, [])
|
|
130
82
|
|
|
131
|
-
// ───
|
|
132
|
-
|
|
133
|
-
const startWebSpeech = useCallback(() => {
|
|
134
|
-
const Ctor = getRecognitionCtor()
|
|
135
|
-
if (!Ctor || recognitionRef.current) return
|
|
136
|
-
setError(null); setNotice(null); setInterim('')
|
|
137
|
-
finalRef.current = ''
|
|
138
|
-
|
|
139
|
-
const rec = new Ctor()
|
|
140
|
-
rec.lang = navigator.language || 'en-US'
|
|
141
|
-
rec.continuous = true
|
|
142
|
-
rec.interimResults = true
|
|
143
|
-
|
|
144
|
-
rec.onresult = (event) => {
|
|
145
|
-
let finalText = ''
|
|
146
|
-
let interimText = ''
|
|
147
|
-
for (let i = 0; i < event.results.length; i++) {
|
|
148
|
-
const res = event.results[i]
|
|
149
|
-
if (res.isFinal) finalText += res[0].transcript
|
|
150
|
-
else interimText += res[0].transcript
|
|
151
|
-
}
|
|
152
|
-
finalRef.current = finalText
|
|
153
|
-
setInterim(interimText)
|
|
154
|
-
}
|
|
83
|
+
// ─── Mic source selection ──────────────────────────
|
|
155
84
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
85
|
+
const refreshDevices = useCallback(async () => {
|
|
86
|
+
if (!navigator.mediaDevices?.enumerateDevices) return
|
|
87
|
+
try {
|
|
88
|
+
const all = await navigator.mediaDevices.enumerateDevices()
|
|
89
|
+
setDevices(all.filter((d) => d.kind === 'audioinput'))
|
|
90
|
+
} catch { /* ignore */ }
|
|
91
|
+
}, [])
|
|
92
|
+
|
|
93
|
+
// Device labels are blank until mic permission is granted, so when the user
|
|
94
|
+
// opens the picker we request a one-shot permission to populate names.
|
|
95
|
+
const ensureDeviceLabels = useCallback(async () => {
|
|
96
|
+
if (!navigator.mediaDevices?.enumerateDevices) return
|
|
97
|
+
try {
|
|
98
|
+
const all = await navigator.mediaDevices.enumerateDevices()
|
|
99
|
+
const inputs = all.filter((d) => d.kind === 'audioinput')
|
|
100
|
+
if (inputs.length && inputs.every((d) => !d.label)) {
|
|
101
|
+
const s = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
102
|
+
s.getTracks().forEach((t) => t.stop())
|
|
163
103
|
}
|
|
164
|
-
}
|
|
104
|
+
} catch { /* permission denied → we'll show generic names */ }
|
|
105
|
+
await refreshDevices()
|
|
106
|
+
}, [refreshDevices])
|
|
165
107
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
108
|
+
const toggleDevicePicker = useCallback(() => {
|
|
109
|
+
setShowDevicePicker((open) => !open)
|
|
110
|
+
}, [])
|
|
111
|
+
|
|
112
|
+
// Populate device labels whenever the picker opens (needs mic permission).
|
|
113
|
+
useEffect(() => {
|
|
114
|
+
if (showDevicePicker) void ensureDeviceLabels()
|
|
115
|
+
}, [showDevicePicker, ensureDeviceLabels])
|
|
173
116
|
|
|
174
|
-
|
|
117
|
+
const chooseDevice = useCallback((id: string | null) => {
|
|
118
|
+
setSelectedDeviceId(id)
|
|
175
119
|
try {
|
|
176
|
-
|
|
177
|
-
|
|
120
|
+
if (id) localStorage.setItem(DEVICE_KEY, id)
|
|
121
|
+
else localStorage.removeItem(DEVICE_KEY)
|
|
122
|
+
} catch { /* ignore */ }
|
|
123
|
+
setShowDevicePicker(false)
|
|
124
|
+
}, [])
|
|
125
|
+
|
|
126
|
+
// Acquire a stream honoring the saved device, gracefully falling back to the
|
|
127
|
+
// OS default if that device was unplugged since it was chosen.
|
|
128
|
+
const getStream = useCallback(async (): Promise<MediaStream> => {
|
|
129
|
+
const id = selectedDeviceIdRef.current
|
|
130
|
+
try {
|
|
131
|
+
return await navigator.mediaDevices.getUserMedia({
|
|
132
|
+
audio: id ? { deviceId: { exact: id } } : true,
|
|
133
|
+
})
|
|
178
134
|
} catch (err) {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
135
|
+
if (id && (err as { name?: string })?.name === 'OverconstrainedError') {
|
|
136
|
+
try { localStorage.removeItem(DEVICE_KEY) } catch { /* ignore */ }
|
|
137
|
+
setSelectedDeviceId(null)
|
|
138
|
+
setNotice('Saved microphone unavailable — using system default.')
|
|
139
|
+
return navigator.mediaDevices.getUserMedia({ audio: true })
|
|
140
|
+
}
|
|
141
|
+
throw err
|
|
183
142
|
}
|
|
184
|
-
}, [switchToLocal])
|
|
185
|
-
|
|
186
|
-
const stopWebSpeech = useCallback(() => {
|
|
187
|
-
try { recognitionRef.current?.stop() } catch { /* already stopped */ }
|
|
188
143
|
}, [])
|
|
189
144
|
|
|
190
|
-
//
|
|
145
|
+
// Keep the list fresh when devices are plugged/unplugged.
|
|
146
|
+
useEffect(() => {
|
|
147
|
+
const md = navigator.mediaDevices
|
|
148
|
+
if (!md?.addEventListener) return
|
|
149
|
+
const onChange = () => { void refreshDevices() }
|
|
150
|
+
md.addEventListener('devicechange', onChange)
|
|
151
|
+
void refreshDevices()
|
|
152
|
+
return () => md.removeEventListener('devicechange', onChange)
|
|
153
|
+
}, [refreshDevices])
|
|
154
|
+
|
|
155
|
+
// Close the picker on an outside click.
|
|
156
|
+
useEffect(() => {
|
|
157
|
+
if (!showDevicePicker) return
|
|
158
|
+
const onDocDown = (e: MouseEvent) => {
|
|
159
|
+
if (!wrapRef.current?.contains(e.target as Node)) setShowDevicePicker(false)
|
|
160
|
+
}
|
|
161
|
+
document.addEventListener('mousedown', onDocDown)
|
|
162
|
+
return () => document.removeEventListener('mousedown', onDocDown)
|
|
163
|
+
}, [showDevicePicker])
|
|
164
|
+
|
|
165
|
+
// ─── On-device (Whisper) transcription ─────────────
|
|
191
166
|
|
|
192
167
|
const handleModelProgress = useCallback((p: ModelProgress) => {
|
|
193
168
|
if (p.status === 'progress' && typeof p.progress === 'number') {
|
|
@@ -203,24 +178,31 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
203
178
|
if (blob.size === 0) { setPhase('idle'); return }
|
|
204
179
|
|
|
205
180
|
setPhase('transcribing')
|
|
206
|
-
|
|
181
|
+
setPartial('')
|
|
207
182
|
try {
|
|
208
|
-
const text = await transcribeBlob(blob, handleModelProgress)
|
|
183
|
+
const text = await transcribeBlob(blob, handleModelProgress, (t) => {
|
|
184
|
+
// First streamed token means the model is loaded and decoding — drop the
|
|
185
|
+
// download indicator and start showing the live transcript.
|
|
186
|
+
setModelPct(null)
|
|
187
|
+
setPartial(t)
|
|
188
|
+
})
|
|
209
189
|
if (text) onTranscriptRef.current(text)
|
|
190
|
+
else setNotice('No speech detected — check the mic source (▾).')
|
|
210
191
|
} catch (err) {
|
|
211
192
|
setError('On-device transcription failed')
|
|
212
193
|
console.error('[voice] local transcription failed', err)
|
|
213
194
|
} finally {
|
|
214
195
|
setPhase('idle')
|
|
215
196
|
setModelPct(null)
|
|
197
|
+
setPartial('')
|
|
216
198
|
}
|
|
217
199
|
}, [stopStream, handleModelProgress])
|
|
218
200
|
|
|
219
201
|
const startLocal = useCallback(async () => {
|
|
220
|
-
setError(null); setNotice(null)
|
|
202
|
+
setError(null); setNotice(null)
|
|
221
203
|
cancelGestureRef.current = false
|
|
222
204
|
try {
|
|
223
|
-
const stream = await
|
|
205
|
+
const stream = await getStream()
|
|
224
206
|
// Released during the permission/await — don't record anything.
|
|
225
207
|
if (cancelGestureRef.current) {
|
|
226
208
|
stream.getTracks().forEach((t) => t.stop())
|
|
@@ -241,7 +223,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
241
223
|
setError(describeMicError(err))
|
|
242
224
|
console.error('[voice] getUserMedia failed', err)
|
|
243
225
|
}
|
|
244
|
-
}, [runLocalTranscription, stopStream])
|
|
226
|
+
}, [runLocalTranscription, stopStream, getStream])
|
|
245
227
|
|
|
246
228
|
const stopLocal = useCallback(() => {
|
|
247
229
|
cancelGestureRef.current = true
|
|
@@ -254,21 +236,18 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
254
236
|
// ─── Push-to-talk gesture ──────────────────────────
|
|
255
237
|
|
|
256
238
|
const handlePointerDown = (e: React.PointerEvent) => {
|
|
257
|
-
if (!
|
|
239
|
+
if (!LOCAL_SUPPORTED || phase !== 'idle') return
|
|
258
240
|
e.preventDefault()
|
|
259
241
|
e.currentTarget.setPointerCapture?.(e.pointerId)
|
|
260
|
-
|
|
261
|
-
else if (LOCAL_SUPPORTED) void startLocal()
|
|
242
|
+
void startLocal()
|
|
262
243
|
}
|
|
263
244
|
const handlePointerUp = (e: React.PointerEvent) => {
|
|
264
245
|
e.currentTarget.releasePointerCapture?.(e.pointerId)
|
|
265
|
-
|
|
266
|
-
else stopLocal()
|
|
246
|
+
stopLocal()
|
|
267
247
|
}
|
|
268
248
|
|
|
269
249
|
// Clean up on unmount.
|
|
270
250
|
useEffect(() => () => {
|
|
271
|
-
try { recognitionRef.current?.abort() } catch { /* ignore */ }
|
|
272
251
|
try { recorderRef.current?.stop() } catch { /* ignore */ }
|
|
273
252
|
streamRef.current?.getTracks().forEach((t) => t.stop())
|
|
274
253
|
}, [])
|
|
@@ -278,33 +257,53 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
278
257
|
const listening = phase === 'listening'
|
|
279
258
|
const busy = phase === 'transcribing'
|
|
280
259
|
|
|
281
|
-
const title = !
|
|
260
|
+
const title = !LOCAL_SUPPORTED
|
|
282
261
|
? 'Voice typing not supported in this build'
|
|
283
262
|
: listening
|
|
284
263
|
? 'Listening… release to insert'
|
|
285
264
|
: busy
|
|
286
265
|
? 'Transcribing…'
|
|
287
|
-
:
|
|
288
|
-
? 'Hold to talk (on-device) — speech is typed into the terminal'
|
|
289
|
-
: 'Hold to talk — speech is typed into the terminal'
|
|
266
|
+
: 'Hold to talk (on-device) — speech is typed into the terminal'
|
|
290
267
|
|
|
291
268
|
let chip: { kind: 'listening' | 'busy' | 'notice' | 'error'; text: string } | null = null
|
|
292
269
|
if (error && phase === 'idle') chip = { kind: 'error', text: error }
|
|
293
270
|
else if (notice && phase === 'idle') chip = { kind: 'notice', text: notice }
|
|
294
|
-
else if (listening) chip = { kind: 'listening', text:
|
|
295
|
-
else if (busy)
|
|
271
|
+
else if (listening) chip = { kind: 'listening', text: 'Recording…' }
|
|
272
|
+
else if (busy) {
|
|
273
|
+
// Show the most recent ~90 chars of the live transcript as a ticker so a long
|
|
274
|
+
// dictation doesn't overflow the chip.
|
|
275
|
+
const preview = partial.length > 90 ? '…' + partial.slice(-90) : partial
|
|
276
|
+
chip = {
|
|
277
|
+
kind: 'busy',
|
|
278
|
+
text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : (preview || 'Transcribing…'),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
296
281
|
|
|
297
282
|
const floating = variant === 'floating'
|
|
298
283
|
const btnClass = floating
|
|
299
284
|
? `voice-btn voice-btn-floating ${listening ? 'listening' : ''} ${busy ? 'busy' : ''}`
|
|
300
285
|
: `titlebar-btn titlebar-icon-btn voice-btn ${listening ? 'listening' : ''} ${busy ? 'busy' : ''}`
|
|
301
286
|
|
|
287
|
+
// Hide Chromium's synthetic "default"/"communications" aggregate entries from
|
|
288
|
+
// the list; we expose our own "System default" item instead.
|
|
289
|
+
const realDevices = devices.filter(
|
|
290
|
+
(d) => d.deviceId && d.deviceId !== 'default' && d.deviceId !== 'communications',
|
|
291
|
+
)
|
|
292
|
+
const rawDefault = devices.find((d) => d.deviceId === 'default')?.label
|
|
293
|
+
const defaultLabel = rawDefault
|
|
294
|
+
? `System default · ${rawDefault.replace(/^Default\s*-\s*/i, '')}`
|
|
295
|
+
: 'System default'
|
|
296
|
+
const activeLabel = selectedDeviceId
|
|
297
|
+
? (realDevices.find((d) => d.deviceId === selectedDeviceId)?.label || 'Selected microphone')
|
|
298
|
+
: defaultLabel
|
|
299
|
+
const showSourcePicker = floating && LOCAL_SUPPORTED
|
|
300
|
+
|
|
302
301
|
return (
|
|
303
|
-
<div className="voice-input-wrap">
|
|
302
|
+
<div className="voice-input-wrap" ref={wrapRef}>
|
|
304
303
|
<button
|
|
305
304
|
type="button"
|
|
306
305
|
className={btnClass}
|
|
307
|
-
disabled={!
|
|
306
|
+
disabled={!LOCAL_SUPPORTED}
|
|
308
307
|
onPointerDown={handlePointerDown}
|
|
309
308
|
onPointerUp={handlePointerUp}
|
|
310
309
|
onPointerCancel={handlePointerUp}
|
|
@@ -315,6 +314,50 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
315
314
|
<span className="voice-icon" aria-hidden="true">🎤</span>
|
|
316
315
|
<span className={`voice-dot ${listening ? 'on' : busy ? 'busy' : 'off'}`} />
|
|
317
316
|
</button>
|
|
317
|
+
{showSourcePicker && (
|
|
318
|
+
<button
|
|
319
|
+
type="button"
|
|
320
|
+
className="voice-source-btn"
|
|
321
|
+
onPointerDown={(e) => e.stopPropagation()}
|
|
322
|
+
onClick={toggleDevicePicker}
|
|
323
|
+
title={`Mic source: ${activeLabel}`}
|
|
324
|
+
aria-label="Choose microphone source"
|
|
325
|
+
aria-expanded={showDevicePicker}
|
|
326
|
+
>
|
|
327
|
+
▾
|
|
328
|
+
</button>
|
|
329
|
+
)}
|
|
330
|
+
{showSourcePicker && showDevicePicker && (
|
|
331
|
+
<div className="voice-source-menu" role="menu">
|
|
332
|
+
<div className="voice-source-head">Microphone source</div>
|
|
333
|
+
<button
|
|
334
|
+
type="button"
|
|
335
|
+
role="menuitemradio"
|
|
336
|
+
aria-checked={selectedDeviceId === null}
|
|
337
|
+
className={`voice-source-item ${selectedDeviceId === null ? 'active' : ''}`}
|
|
338
|
+
onClick={() => chooseDevice(null)}
|
|
339
|
+
>
|
|
340
|
+
<span className="voice-source-check">{selectedDeviceId === null ? '✓' : ''}</span>
|
|
341
|
+
<span className="voice-source-label">{defaultLabel}</span>
|
|
342
|
+
</button>
|
|
343
|
+
{realDevices.map((d, i) => (
|
|
344
|
+
<button
|
|
345
|
+
key={d.deviceId}
|
|
346
|
+
type="button"
|
|
347
|
+
role="menuitemradio"
|
|
348
|
+
aria-checked={selectedDeviceId === d.deviceId}
|
|
349
|
+
className={`voice-source-item ${selectedDeviceId === d.deviceId ? 'active' : ''}`}
|
|
350
|
+
onClick={() => chooseDevice(d.deviceId)}
|
|
351
|
+
>
|
|
352
|
+
<span className="voice-source-check">{selectedDeviceId === d.deviceId ? '✓' : ''}</span>
|
|
353
|
+
<span className="voice-source-label">{d.label || `Microphone ${i + 1}`}</span>
|
|
354
|
+
</button>
|
|
355
|
+
))}
|
|
356
|
+
{realDevices.length === 0 && (
|
|
357
|
+
<div className="voice-source-empty">No microphones found</div>
|
|
358
|
+
)}
|
|
359
|
+
</div>
|
|
360
|
+
)}
|
|
318
361
|
{chip && <div className={`voice-chip ${chip.kind} ${floating ? 'voice-chip-floating' : ''}`}>{chip.text}</div>}
|
|
319
362
|
</div>
|
|
320
363
|
)
|
package/src/renderer/index.html
CHANGED
|
@@ -4,9 +4,71 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>ctlsurf-worker</title>
|
|
7
|
+
<!-- Boot splash styles are inlined here so they paint on the first frame,
|
|
8
|
+
before the JS bundle and styles.css have loaded. Without this the window
|
|
9
|
+
shows an empty dark pane while React mounts and the startup IPC runs. -->
|
|
10
|
+
<style>
|
|
11
|
+
#boot-splash {
|
|
12
|
+
position: fixed;
|
|
13
|
+
inset: 0;
|
|
14
|
+
z-index: 9999;
|
|
15
|
+
display: flex;
|
|
16
|
+
flex-direction: column;
|
|
17
|
+
align-items: center;
|
|
18
|
+
justify-content: center;
|
|
19
|
+
gap: 16px;
|
|
20
|
+
background: #1a1b26;
|
|
21
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
22
|
+
transition: opacity 0.3s ease;
|
|
23
|
+
-webkit-app-region: drag; /* keep the frameless window draggable during load */
|
|
24
|
+
}
|
|
25
|
+
#boot-splash.boot-splash-hidden {
|
|
26
|
+
opacity: 0;
|
|
27
|
+
pointer-events: none;
|
|
28
|
+
}
|
|
29
|
+
.boot-spinner {
|
|
30
|
+
width: 34px;
|
|
31
|
+
height: 34px;
|
|
32
|
+
border-radius: 50%;
|
|
33
|
+
border: 3px solid rgba(122, 162, 247, 0.22);
|
|
34
|
+
border-top-color: #7aa2f7;
|
|
35
|
+
animation: boot-spin 0.8s linear infinite;
|
|
36
|
+
}
|
|
37
|
+
@keyframes boot-spin {
|
|
38
|
+
to { transform: rotate(360deg); }
|
|
39
|
+
}
|
|
40
|
+
.boot-splash-title {
|
|
41
|
+
font-size: 15px;
|
|
42
|
+
font-weight: 600;
|
|
43
|
+
letter-spacing: 0.02em;
|
|
44
|
+
color: #c0caf5;
|
|
45
|
+
}
|
|
46
|
+
.boot-splash-label {
|
|
47
|
+
font-size: 12px;
|
|
48
|
+
letter-spacing: 0.04em;
|
|
49
|
+
color: #565f89;
|
|
50
|
+
}
|
|
51
|
+
</style>
|
|
7
52
|
</head>
|
|
8
53
|
<body>
|
|
54
|
+
<div id="boot-splash">
|
|
55
|
+
<div class="boot-spinner"></div>
|
|
56
|
+
<div class="boot-splash-title">ctlsurf</div>
|
|
57
|
+
<div class="boot-splash-label">Starting up…</div>
|
|
58
|
+
</div>
|
|
9
59
|
<div id="root"></div>
|
|
60
|
+
<script>
|
|
61
|
+
// Fades out and removes the boot splash. Called by the React app once its
|
|
62
|
+
// startup IPC has settled (see App.tsx). The timeout is a safety net so a
|
|
63
|
+
// hung startup never leaves the user stuck behind the splash forever.
|
|
64
|
+
window.__dismissSplash = function () {
|
|
65
|
+
var el = document.getElementById('boot-splash')
|
|
66
|
+
if (!el) return
|
|
67
|
+
el.classList.add('boot-splash-hidden')
|
|
68
|
+
setTimeout(function () { el.remove() }, 350)
|
|
69
|
+
}
|
|
70
|
+
setTimeout(function () { window.__dismissSplash() }, 10000)
|
|
71
|
+
</script>
|
|
10
72
|
<script type="module" src="./main.tsx"></script>
|
|
11
73
|
</body>
|
|
12
74
|
</html>
|
|
@@ -6,6 +6,37 @@
|
|
|
6
6
|
const MODEL = 'Xenova/whisper-base'
|
|
7
7
|
const TARGET_SAMPLE_RATE = 16000
|
|
8
8
|
|
|
9
|
+
// Whisper hallucinates filler tokens ("you", "Thank you.", "Thanks for
|
|
10
|
+
// watching.") when fed silence. Two guards below tame that:
|
|
11
|
+
// - Below SILENCE_RMS we treat the clip as silent and skip the model entirely.
|
|
12
|
+
// - Between SILENCE_RMS and LOW_CONFIDENCE_RMS we still run the model but drop
|
|
13
|
+
// the result if it's *only* a known filler phrase (likely a hallucination on
|
|
14
|
+
// a quiet clip rather than real speech).
|
|
15
|
+
const SILENCE_RMS = 0.008
|
|
16
|
+
const LOW_CONFIDENCE_RMS = 0.02
|
|
17
|
+
const FILLER_PHRASES = new Set([
|
|
18
|
+
'you',
|
|
19
|
+
'thank you',
|
|
20
|
+
'thank you very much',
|
|
21
|
+
'thank you for watching',
|
|
22
|
+
'thanks for watching',
|
|
23
|
+
'please subscribe',
|
|
24
|
+
])
|
|
25
|
+
|
|
26
|
+
// Root-mean-square amplitude of the clip — a cheap loudness proxy. Normalized
|
|
27
|
+
// speech sits around 0.05–0.15; a silent room is well under 0.005.
|
|
28
|
+
function computeRms(pcm: Float32Array): number {
|
|
29
|
+
if (pcm.length === 0) return 0
|
|
30
|
+
let sum = 0
|
|
31
|
+
for (let i = 0; i < pcm.length; i++) sum += pcm[i] * pcm[i]
|
|
32
|
+
return Math.sqrt(sum / pcm.length)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function isFillerOnly(text: string): boolean {
|
|
36
|
+
const norm = text.toLowerCase().replace(/[.!?,…"']/g, '').replace(/\s+/g, ' ').trim()
|
|
37
|
+
return FILLER_PHRASES.has(norm)
|
|
38
|
+
}
|
|
39
|
+
|
|
9
40
|
export interface ModelProgress {
|
|
10
41
|
status: string
|
|
11
42
|
file?: string
|
|
@@ -76,13 +107,70 @@ async function blobToPcm16k(blob: Blob): Promise<Float32Array | null> {
|
|
|
76
107
|
return rendered.getChannelData(0)
|
|
77
108
|
}
|
|
78
109
|
|
|
79
|
-
export async function transcribeBlob(
|
|
80
|
-
|
|
110
|
+
export async function transcribeBlob(
|
|
111
|
+
blob: Blob,
|
|
112
|
+
onProgress?: (p: ModelProgress) => void,
|
|
113
|
+
// Optional live preview: called with the transcript-so-far as the model
|
|
114
|
+
// decodes, for progressive on-screen feedback. The return value (below) is
|
|
115
|
+
// still the authoritative, de-duplicated text used for the actual insert.
|
|
116
|
+
onPartial?: (text: string) => void,
|
|
117
|
+
): Promise<string> {
|
|
118
|
+
// Decode first so a silent clip never triggers the (one-time, ~75MB) model
|
|
119
|
+
// download just to be thrown away below.
|
|
81
120
|
const pcm = await blobToPcm16k(blob)
|
|
82
121
|
if (!pcm) return ''
|
|
83
|
-
const
|
|
84
|
-
|
|
122
|
+
const rms = computeRms(pcm)
|
|
123
|
+
if (rms < SILENCE_RMS) {
|
|
124
|
+
console.info(`[voice] near-silent clip (rms=${rms.toFixed(4)}); skipping transcription`)
|
|
125
|
+
return ''
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const transcriber = await loadTranscriber(onProgress)
|
|
129
|
+
|
|
130
|
+
// Stream decoded text to onPartial as it's generated. `committed` holds text
|
|
131
|
+
// from finished chunks; `current` is the in-progress chunk — keeping them
|
|
132
|
+
// separate avoids the stride overlap duplicating words across 30s boundaries
|
|
133
|
+
// in the preview. Best-effort: if the streamer can't be built we just skip the
|
|
134
|
+
// preview and still return the final text.
|
|
135
|
+
let streamer: unknown
|
|
136
|
+
if (onPartial) {
|
|
137
|
+
try {
|
|
138
|
+
const { WhisperTextStreamer } = await import('@huggingface/transformers')
|
|
139
|
+
const tokenizer = (transcriber as unknown as {
|
|
140
|
+
tokenizer: ConstructorParameters<typeof WhisperTextStreamer>[0]
|
|
141
|
+
}).tokenizer
|
|
142
|
+
let committed = ''
|
|
143
|
+
let current = ''
|
|
144
|
+
streamer = new WhisperTextStreamer(tokenizer, {
|
|
145
|
+
on_chunk_start: () => { current = '' },
|
|
146
|
+
callback_function: (delta: string) => {
|
|
147
|
+
current += delta
|
|
148
|
+
onPartial((committed + current).replace(/\s+/g, ' ').trim())
|
|
149
|
+
},
|
|
150
|
+
on_chunk_end: () => { committed = (committed + current).trim() + ' '; current = '' },
|
|
151
|
+
})
|
|
152
|
+
} catch (err) {
|
|
153
|
+
console.warn('[voice] streaming preview unavailable', err)
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Whisper only attends to a 30s window at a time. Without chunk_length_s the
|
|
158
|
+
// pipeline truncates to that first window, silently dropping anything spoken
|
|
159
|
+
// past ~30s. Chunking transcribes the full clip in overlapping 30s windows
|
|
160
|
+
// (stride gives overlap so words aren't lost at the seams) so long speeches
|
|
161
|
+
// are captured in full.
|
|
162
|
+
const result = await transcriber(pcm, {
|
|
163
|
+
chunk_length_s: 30,
|
|
164
|
+
stride_length_s: 5,
|
|
165
|
+
...(streamer ? { streamer } : {}),
|
|
166
|
+
})
|
|
167
|
+
const text = (Array.isArray(result)
|
|
85
168
|
? result.map((r) => r.text).join(' ')
|
|
86
|
-
: result?.text
|
|
87
|
-
|
|
169
|
+
: result?.text || '').trim()
|
|
170
|
+
|
|
171
|
+
if (text && rms < LOW_CONFIDENCE_RMS && isFillerOnly(text)) {
|
|
172
|
+
console.info(`[voice] dropping filler-only output "${text}" from quiet clip (rms=${rms.toFixed(4)})`)
|
|
173
|
+
return ''
|
|
174
|
+
}
|
|
175
|
+
return text
|
|
88
176
|
}
|