@phenx-inc/ctlsurf 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/out/headless/index.mjs +169 -10
  2. package/out/headless/index.mjs.map +2 -2
  3. package/out/main/index.js +175 -9
  4. package/out/preload/index.js +10 -0
  5. package/out/renderer/assets/{cssMode-eTXVdAkZ.js → cssMode-BdYp0fp9.js} +3 -3
  6. package/out/renderer/assets/{freemarker2-B5BKaiK4.js → freemarker2-Cya-9zr-.js} +1 -1
  7. package/out/renderer/assets/{handlebars-BIdLd2wU.js → handlebars-EJs9W5Ub.js} +1 -1
  8. package/out/renderer/assets/{html-BXL4cnLS.js → html-Ex4qy2lh.js} +1 -1
  9. package/out/renderer/assets/{htmlMode-46N3XG2c.js → htmlMode-CCm0aJ6C.js} +3 -3
  10. package/out/renderer/assets/{index-dRvutfbl.js → index-Cj7K9-Ba.js} +823 -137
  11. package/out/renderer/assets/{index-Cf-RsxoC.css → index-nvsNYP5c.css} +229 -0
  12. package/out/renderer/assets/{javascript-n_iZZzDX.js → javascript-B2niCouI.js} +2 -2
  13. package/out/renderer/assets/{jsonMode-DXDczSNu.js → jsonMode-BgCUHaFo.js} +3 -3
  14. package/out/renderer/assets/{liquid-B1QweUh7.js → liquid-DOQmSo4G.js} +1 -1
  15. package/out/renderer/assets/{lspLanguageFeatures-DqzMqkRk.js → lspLanguageFeatures-8SIcRAMh.js} +1 -1
  16. package/out/renderer/assets/{mdx-BCv8lm5e.js → mdx-DuKdGNKE.js} +1 -1
  17. package/out/renderer/assets/{python-BLNzYwDv.js → python-BUa4XFG-.js} +1 -1
  18. package/out/renderer/assets/{razor-CvAww8bG.js → razor-BUw8jho0.js} +1 -1
  19. package/out/renderer/assets/{transformers.web-DtSCnG36.js → transformers.web-_PuZoIho.js} +824 -19
  20. package/out/renderer/assets/{tsMode-C7m6Kr5E.js → tsMode-8FLRnobw.js} +1 -1
  21. package/out/renderer/assets/{typescript-DhPw4VVg.js → typescript-rdwAjCi4.js} +1 -1
  22. package/out/renderer/assets/{xml-B0WLFJ2U.js → xml-HOzT59Qg.js} +1 -1
  23. package/out/renderer/assets/{yaml-BWyn9Wd7.js → yaml-B81JfLsW.js} +1 -1
  24. package/out/renderer/index.html +64 -2
  25. package/out/repro-toggle.mjs +305 -0
  26. package/package.json +1 -1
  27. package/src/main/index.ts +9 -0
  28. package/src/main/orchestrator.ts +51 -10
  29. package/src/main/transcripts.ts +134 -0
  30. package/src/preload/index.ts +14 -0
  31. package/src/renderer/App.tsx +30 -11
  32. package/src/renderer/components/FloatingMic.tsx +48 -0
  33. package/src/renderer/components/SpeakControls.tsx +235 -0
  34. package/src/renderer/components/VoiceInput.tsx +183 -140
  35. package/src/renderer/index.html +62 -0
  36. package/src/renderer/lib/localWhisper.ts +94 -6
  37. package/src/renderer/lib/speech.ts +317 -0
  38. package/src/renderer/styles.css +229 -0
@@ -1,70 +1,26 @@
1
1
  import { useCallback, useEffect, useRef, useState } from 'react'
2
2
  import { transcribeBlob, type ModelProgress } from '../lib/localWhisper'
3
3
 
4
- // ─── Minimal Web Speech API typings ──────────────────
5
- // webkitSpeechRecognition isn't in the standard DOM lib, so declare just the
6
- // surface we use. This API is frequently unavailable inside Electron (Chromium
7
- // ships without Google's speech backend); when it fails we fall back to a local
8
- // Whisper model (see ../lib/localWhisper).
9
-
10
- interface SpeechRecognitionResult {
11
- isFinal: boolean
12
- 0: { transcript: string }
13
- }
14
- interface SpeechRecognitionEvent {
15
- resultIndex: number
16
- results: { length: number;[index: number]: SpeechRecognitionResult }
17
- }
18
- interface SpeechRecognitionErrorEvent { error: string }
19
- interface SpeechRecognitionLike {
20
- lang: string
21
- continuous: boolean
22
- interimResults: boolean
23
- start: () => void
24
- stop: () => void
25
- abort: () => void
26
- onresult: ((e: SpeechRecognitionEvent) => void) | null
27
- onerror: ((e: SpeechRecognitionErrorEvent) => void) | null
28
- onend: (() => void) | null
29
- }
30
- type SpeechRecognitionCtor = new () => SpeechRecognitionLike
31
-
32
- function getRecognitionCtor(): SpeechRecognitionCtor | null {
33
- const w = window as unknown as {
34
- SpeechRecognition?: SpeechRecognitionCtor
35
- webkitSpeechRecognition?: SpeechRecognitionCtor
36
- }
37
- return w.SpeechRecognition || w.webkitSpeechRecognition || null
38
- }
4
+ // ─── Capabilities ────────────────────────────────────
5
+ // Speech is transcribed entirely on-device via a local Whisper model (see
6
+ // ../lib/localWhisper). The browser's Web Speech API is intentionally not used:
7
+ // it's a cloud service whose Google backend is unreachable inside Electron
8
+ // (Chromium ships without the API key), so it never works in this app.
39
9
 
40
- // ─── Capabilities & engine selection ─────────────────
41
-
42
- type Engine = 'web-speech' | 'local'
43
10
  type Phase = 'idle' | 'listening' | 'transcribing'
44
11
 
45
- const ENGINE_KEY = 'ctlsurf.voiceEngine'
12
+ // Persisted audioinput deviceId; null/absent = OS default.
13
+ const DEVICE_KEY = 'ctlsurf.voiceDeviceId'
14
+
15
+ function loadDeviceId(): string | null {
16
+ try { return localStorage.getItem(DEVICE_KEY) || null } catch { return null }
17
+ }
46
18
 
47
- const WEB_SPEECH_SUPPORTED = getRecognitionCtor() !== null
48
19
  const LOCAL_SUPPORTED =
49
20
  typeof navigator !== 'undefined' &&
50
21
  !!navigator.mediaDevices?.getUserMedia &&
51
22
  typeof MediaRecorder !== 'undefined' &&
52
23
  typeof OfflineAudioContext !== 'undefined'
53
- const ANY_SUPPORTED = WEB_SPEECH_SUPPORTED || LOCAL_SUPPORTED
54
-
55
- function loadInitialEngine(): Engine {
56
- if (!WEB_SPEECH_SUPPORTED && LOCAL_SUPPORTED) return 'local'
57
- try {
58
- if (localStorage.getItem(ENGINE_KEY) === 'local' && LOCAL_SUPPORTED) return 'local'
59
- } catch { /* ignore */ }
60
- return WEB_SPEECH_SUPPORTED ? 'web-speech' : 'local'
61
- }
62
-
63
- // Web Speech errors that mean the engine itself is unreachable (vs. a mic
64
- // permission/hardware problem, which would also break the local fallback).
65
- function isEngineUnavailable(code: string): boolean {
66
- return code === 'network' || code === 'service-not-allowed'
67
- }
68
24
 
69
25
  function describeMicError(err: unknown): string {
70
26
  const name = (err as { name?: string })?.name
@@ -82,26 +38,28 @@ interface VoiceInputProps {
82
38
  }
83
39
 
84
40
  export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputProps) {
85
- const [engine, setEngine] = useState<Engine>(loadInitialEngine)
86
41
  const [phase, setPhase] = useState<Phase>('idle')
87
- const [interim, setInterim] = useState('')
88
42
  const [modelPct, setModelPct] = useState<number | null>(null)
43
+ // Transcript-so-far streamed from the model during transcription (live preview).
44
+ const [partial, setPartial] = useState('')
89
45
  const [error, setError] = useState<string | null>(null)
90
46
  const [notice, setNotice] = useState<string | null>(null)
91
47
 
92
- // Web Speech refs
93
- const recognitionRef = useRef<SpeechRecognitionLike | null>(null)
94
- const finalRef = useRef('')
95
- // Local (Whisper) refs
48
+ // Mic source selection
49
+ const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
50
+ const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(loadDeviceId)
51
+ const [showDevicePicker, setShowDevicePicker] = useState(false)
52
+ const selectedDeviceIdRef = useRef(selectedDeviceId)
53
+ useEffect(() => { selectedDeviceIdRef.current = selectedDeviceId }, [selectedDeviceId])
54
+ const wrapRef = useRef<HTMLDivElement>(null)
55
+
56
+ // Recording refs
96
57
  const streamRef = useRef<MediaStream | null>(null)
97
58
  const recorderRef = useRef<MediaRecorder | null>(null)
98
59
  const chunksRef = useRef<Blob[]>([])
99
60
  // Set true when the user releases before getUserMedia resolves (quick tap).
100
61
  const cancelGestureRef = useRef(false)
101
62
 
102
- const engineRef = useRef(engine)
103
- useEffect(() => { engineRef.current = engine }, [engine])
104
-
105
63
  const onTranscriptRef = useRef(onTranscript)
106
64
  useEffect(() => { onTranscriptRef.current = onTranscript }, [onTranscript])
107
65
 
@@ -117,77 +75,94 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
117
75
  return () => clearTimeout(t)
118
76
  }, [notice])
119
77
 
120
- const switchToLocal = useCallback((reason: string) => {
121
- try { localStorage.setItem(ENGINE_KEY, 'local') } catch { /* ignore */ }
122
- setEngine('local')
123
- setNotice(reason)
124
- }, [])
125
-
126
78
  const stopStream = useCallback(() => {
127
79
  streamRef.current?.getTracks().forEach((t) => t.stop())
128
80
  streamRef.current = null
129
81
  }, [])
130
82
 
131
- // ─── Web Speech engine ─────────────────────────────
132
-
133
- const startWebSpeech = useCallback(() => {
134
- const Ctor = getRecognitionCtor()
135
- if (!Ctor || recognitionRef.current) return
136
- setError(null); setNotice(null); setInterim('')
137
- finalRef.current = ''
138
-
139
- const rec = new Ctor()
140
- rec.lang = navigator.language || 'en-US'
141
- rec.continuous = true
142
- rec.interimResults = true
143
-
144
- rec.onresult = (event) => {
145
- let finalText = ''
146
- let interimText = ''
147
- for (let i = 0; i < event.results.length; i++) {
148
- const res = event.results[i]
149
- if (res.isFinal) finalText += res[0].transcript
150
- else interimText += res[0].transcript
151
- }
152
- finalRef.current = finalText
153
- setInterim(interimText)
154
- }
83
+ // ─── Mic source selection ──────────────────────────
155
84
 
156
- rec.onerror = (event) => {
157
- if (isEngineUnavailable(event.error) && LOCAL_SUPPORTED) {
158
- // The streamed audio is gone; switch engines and ask for a retry.
159
- finalRef.current = ''
160
- switchToLocal('Voice service unavailable — switched to on-device. Press again.')
161
- } else if (event.error !== 'no-speech' && event.error !== 'aborted') {
162
- setError(event.error === 'not-allowed' ? 'Microphone access denied' : `Voice error: ${event.error}`)
85
+ const refreshDevices = useCallback(async () => {
86
+ if (!navigator.mediaDevices?.enumerateDevices) return
87
+ try {
88
+ const all = await navigator.mediaDevices.enumerateDevices()
89
+ setDevices(all.filter((d) => d.kind === 'audioinput'))
90
+ } catch { /* ignore */ }
91
+ }, [])
92
+
93
+ // Device labels are blank until mic permission is granted, so when the user
94
+ // opens the picker we request a one-shot permission to populate names.
95
+ const ensureDeviceLabels = useCallback(async () => {
96
+ if (!navigator.mediaDevices?.enumerateDevices) return
97
+ try {
98
+ const all = await navigator.mediaDevices.enumerateDevices()
99
+ const inputs = all.filter((d) => d.kind === 'audioinput')
100
+ if (inputs.length && inputs.every((d) => !d.label)) {
101
+ const s = await navigator.mediaDevices.getUserMedia({ audio: true })
102
+ s.getTracks().forEach((t) => t.stop())
163
103
  }
164
- }
104
+ } catch { /* permission denied → we'll show generic names */ }
105
+ await refreshDevices()
106
+ }, [refreshDevices])
165
107
 
166
- rec.onend = () => {
167
- const text = finalRef.current.trim()
168
- recognitionRef.current = null
169
- setPhase('idle')
170
- setInterim('')
171
- if (text) onTranscriptRef.current(text)
172
- }
108
+ const toggleDevicePicker = useCallback(() => {
109
+ setShowDevicePicker((open) => !open)
110
+ }, [])
111
+
112
+ // Populate device labels whenever the picker opens (needs mic permission).
113
+ useEffect(() => {
114
+ if (showDevicePicker) void ensureDeviceLabels()
115
+ }, [showDevicePicker, ensureDeviceLabels])
173
116
 
174
- recognitionRef.current = rec
117
+ const chooseDevice = useCallback((id: string | null) => {
118
+ setSelectedDeviceId(id)
175
119
  try {
176
- rec.start()
177
- setPhase('listening')
120
+ if (id) localStorage.setItem(DEVICE_KEY, id)
121
+ else localStorage.removeItem(DEVICE_KEY)
122
+ } catch { /* ignore */ }
123
+ setShowDevicePicker(false)
124
+ }, [])
125
+
126
+ // Acquire a stream honoring the saved device, gracefully falling back to the
127
+ // OS default if that device was unplugged since it was chosen.
128
+ const getStream = useCallback(async (): Promise<MediaStream> => {
129
+ const id = selectedDeviceIdRef.current
130
+ try {
131
+ return await navigator.mediaDevices.getUserMedia({
132
+ audio: id ? { deviceId: { exact: id } } : true,
133
+ })
178
134
  } catch (err) {
179
- recognitionRef.current = null
180
- setPhase('idle')
181
- setError('Could not start microphone')
182
- console.error('[voice] web speech start failed', err)
135
+ if (id && (err as { name?: string })?.name === 'OverconstrainedError') {
136
+ try { localStorage.removeItem(DEVICE_KEY) } catch { /* ignore */ }
137
+ setSelectedDeviceId(null)
138
+ setNotice('Saved microphone unavailable — using system default.')
139
+ return navigator.mediaDevices.getUserMedia({ audio: true })
140
+ }
141
+ throw err
183
142
  }
184
- }, [switchToLocal])
185
-
186
- const stopWebSpeech = useCallback(() => {
187
- try { recognitionRef.current?.stop() } catch { /* already stopped */ }
188
143
  }, [])
189
144
 
190
- // ─── Local (Whisper) engine ────────────────────────
145
+ // Keep the list fresh when devices are plugged/unplugged.
146
+ useEffect(() => {
147
+ const md = navigator.mediaDevices
148
+ if (!md?.addEventListener) return
149
+ const onChange = () => { void refreshDevices() }
150
+ md.addEventListener('devicechange', onChange)
151
+ void refreshDevices()
152
+ return () => md.removeEventListener('devicechange', onChange)
153
+ }, [refreshDevices])
154
+
155
+ // Close the picker on an outside click.
156
+ useEffect(() => {
157
+ if (!showDevicePicker) return
158
+ const onDocDown = (e: MouseEvent) => {
159
+ if (!wrapRef.current?.contains(e.target as Node)) setShowDevicePicker(false)
160
+ }
161
+ document.addEventListener('mousedown', onDocDown)
162
+ return () => document.removeEventListener('mousedown', onDocDown)
163
+ }, [showDevicePicker])
164
+
165
+ // ─── On-device (Whisper) transcription ─────────────
191
166
 
192
167
  const handleModelProgress = useCallback((p: ModelProgress) => {
193
168
  if (p.status === 'progress' && typeof p.progress === 'number') {
@@ -203,24 +178,31 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
203
178
  if (blob.size === 0) { setPhase('idle'); return }
204
179
 
205
180
  setPhase('transcribing')
206
- setInterim('')
181
+ setPartial('')
207
182
  try {
208
- const text = await transcribeBlob(blob, handleModelProgress)
183
+ const text = await transcribeBlob(blob, handleModelProgress, (t) => {
184
+ // First streamed token means the model is loaded and decoding — drop the
185
+ // download indicator and start showing the live transcript.
186
+ setModelPct(null)
187
+ setPartial(t)
188
+ })
209
189
  if (text) onTranscriptRef.current(text)
190
+ else setNotice('No speech detected — check the mic source (▾).')
210
191
  } catch (err) {
211
192
  setError('On-device transcription failed')
212
193
  console.error('[voice] local transcription failed', err)
213
194
  } finally {
214
195
  setPhase('idle')
215
196
  setModelPct(null)
197
+ setPartial('')
216
198
  }
217
199
  }, [stopStream, handleModelProgress])
218
200
 
219
201
  const startLocal = useCallback(async () => {
220
- setError(null); setNotice(null); setInterim('')
202
+ setError(null); setNotice(null)
221
203
  cancelGestureRef.current = false
222
204
  try {
223
- const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
205
+ const stream = await getStream()
224
206
  // Released during the permission/await — don't record anything.
225
207
  if (cancelGestureRef.current) {
226
208
  stream.getTracks().forEach((t) => t.stop())
@@ -241,7 +223,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
241
223
  setError(describeMicError(err))
242
224
  console.error('[voice] getUserMedia failed', err)
243
225
  }
244
- }, [runLocalTranscription, stopStream])
226
+ }, [runLocalTranscription, stopStream, getStream])
245
227
 
246
228
  const stopLocal = useCallback(() => {
247
229
  cancelGestureRef.current = true
@@ -254,21 +236,18 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
254
236
  // ─── Push-to-talk gesture ──────────────────────────
255
237
 
256
238
  const handlePointerDown = (e: React.PointerEvent) => {
257
- if (!ANY_SUPPORTED || phase !== 'idle') return
239
+ if (!LOCAL_SUPPORTED || phase !== 'idle') return
258
240
  e.preventDefault()
259
241
  e.currentTarget.setPointerCapture?.(e.pointerId)
260
- if (engineRef.current === 'web-speech' && WEB_SPEECH_SUPPORTED) startWebSpeech()
261
- else if (LOCAL_SUPPORTED) void startLocal()
242
+ void startLocal()
262
243
  }
263
244
  const handlePointerUp = (e: React.PointerEvent) => {
264
245
  e.currentTarget.releasePointerCapture?.(e.pointerId)
265
- if (engineRef.current === 'web-speech') stopWebSpeech()
266
- else stopLocal()
246
+ stopLocal()
267
247
  }
268
248
 
269
249
  // Clean up on unmount.
270
250
  useEffect(() => () => {
271
- try { recognitionRef.current?.abort() } catch { /* ignore */ }
272
251
  try { recorderRef.current?.stop() } catch { /* ignore */ }
273
252
  streamRef.current?.getTracks().forEach((t) => t.stop())
274
253
  }, [])
@@ -278,33 +257,53 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
278
257
  const listening = phase === 'listening'
279
258
  const busy = phase === 'transcribing'
280
259
 
281
- const title = !ANY_SUPPORTED
260
+ const title = !LOCAL_SUPPORTED
282
261
  ? 'Voice typing not supported in this build'
283
262
  : listening
284
263
  ? 'Listening… release to insert'
285
264
  : busy
286
265
  ? 'Transcribing…'
287
- : engine === 'local'
288
- ? 'Hold to talk (on-device) — speech is typed into the terminal'
289
- : 'Hold to talk — speech is typed into the terminal'
266
+ : 'Hold to talk (on-device) — speech is typed into the terminal'
290
267
 
291
268
  let chip: { kind: 'listening' | 'busy' | 'notice' | 'error'; text: string } | null = null
292
269
  if (error && phase === 'idle') chip = { kind: 'error', text: error }
293
270
  else if (notice && phase === 'idle') chip = { kind: 'notice', text: notice }
294
- else if (listening) chip = { kind: 'listening', text: interim || (engine === 'local' ? 'Recording…' : 'Listening…') }
295
- else if (busy) chip = { kind: 'busy', text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : 'Transcribing…' }
271
+ else if (listening) chip = { kind: 'listening', text: 'Recording…' }
272
+ else if (busy) {
273
+ // Show the most recent ~90 chars of the live transcript as a ticker so a long
274
+ // dictation doesn't overflow the chip.
275
+ const preview = partial.length > 90 ? '…' + partial.slice(-90) : partial
276
+ chip = {
277
+ kind: 'busy',
278
+ text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : (preview || 'Transcribing…'),
279
+ }
280
+ }
296
281
 
297
282
  const floating = variant === 'floating'
298
283
  const btnClass = floating
299
284
  ? `voice-btn voice-btn-floating ${listening ? 'listening' : ''} ${busy ? 'busy' : ''}`
300
285
  : `titlebar-btn titlebar-icon-btn voice-btn ${listening ? 'listening' : ''} ${busy ? 'busy' : ''}`
301
286
 
287
+ // Hide Chromium's synthetic "default"/"communications" aggregate entries from
288
+ // the list; we expose our own "System default" item instead.
289
+ const realDevices = devices.filter(
290
+ (d) => d.deviceId && d.deviceId !== 'default' && d.deviceId !== 'communications',
291
+ )
292
+ const rawDefault = devices.find((d) => d.deviceId === 'default')?.label
293
+ const defaultLabel = rawDefault
294
+ ? `System default · ${rawDefault.replace(/^Default\s*-\s*/i, '')}`
295
+ : 'System default'
296
+ const activeLabel = selectedDeviceId
297
+ ? (realDevices.find((d) => d.deviceId === selectedDeviceId)?.label || 'Selected microphone')
298
+ : defaultLabel
299
+ const showSourcePicker = floating && LOCAL_SUPPORTED
300
+
302
301
  return (
303
- <div className="voice-input-wrap">
302
+ <div className="voice-input-wrap" ref={wrapRef}>
304
303
  <button
305
304
  type="button"
306
305
  className={btnClass}
307
- disabled={!ANY_SUPPORTED}
306
+ disabled={!LOCAL_SUPPORTED}
308
307
  onPointerDown={handlePointerDown}
309
308
  onPointerUp={handlePointerUp}
310
309
  onPointerCancel={handlePointerUp}
@@ -315,6 +314,50 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
315
314
  <span className="voice-icon" aria-hidden="true">🎤</span>
316
315
  <span className={`voice-dot ${listening ? 'on' : busy ? 'busy' : 'off'}`} />
317
316
  </button>
317
+ {showSourcePicker && (
318
+ <button
319
+ type="button"
320
+ className="voice-source-btn"
321
+ onPointerDown={(e) => e.stopPropagation()}
322
+ onClick={toggleDevicePicker}
323
+ title={`Mic source: ${activeLabel}`}
324
+ aria-label="Choose microphone source"
325
+ aria-expanded={showDevicePicker}
326
+ >
327
+ ▾
328
+ </button>
329
+ )}
330
+ {showSourcePicker && showDevicePicker && (
331
+ <div className="voice-source-menu" role="menu">
332
+ <div className="voice-source-head">Microphone source</div>
333
+ <button
334
+ type="button"
335
+ role="menuitemradio"
336
+ aria-checked={selectedDeviceId === null}
337
+ className={`voice-source-item ${selectedDeviceId === null ? 'active' : ''}`}
338
+ onClick={() => chooseDevice(null)}
339
+ >
340
+ <span className="voice-source-check">{selectedDeviceId === null ? '✓' : ''}</span>
341
+ <span className="voice-source-label">{defaultLabel}</span>
342
+ </button>
343
+ {realDevices.map((d, i) => (
344
+ <button
345
+ key={d.deviceId}
346
+ type="button"
347
+ role="menuitemradio"
348
+ aria-checked={selectedDeviceId === d.deviceId}
349
+ className={`voice-source-item ${selectedDeviceId === d.deviceId ? 'active' : ''}`}
350
+ onClick={() => chooseDevice(d.deviceId)}
351
+ >
352
+ <span className="voice-source-check">{selectedDeviceId === d.deviceId ? '✓' : ''}</span>
353
+ <span className="voice-source-label">{d.label || `Microphone ${i + 1}`}</span>
354
+ </button>
355
+ ))}
356
+ {realDevices.length === 0 && (
357
+ <div className="voice-source-empty">No microphones found</div>
358
+ )}
359
+ </div>
360
+ )}
318
361
  {chip && <div className={`voice-chip ${chip.kind} ${floating ? 'voice-chip-floating' : ''}`}>{chip.text}</div>}
319
362
  </div>
320
363
  )
@@ -4,9 +4,71 @@
4
4
  <meta charset="UTF-8" />
5
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
6
  <title>ctlsurf-worker</title>
7
+ <!-- Boot splash styles are inlined here so they paint on the first frame,
8
+ before the JS bundle and styles.css have loaded. Without this the window
9
+ shows an empty dark pane while React mounts and the startup IPC runs. -->
10
+ <style>
11
+ #boot-splash {
12
+ position: fixed;
13
+ inset: 0;
14
+ z-index: 9999;
15
+ display: flex;
16
+ flex-direction: column;
17
+ align-items: center;
18
+ justify-content: center;
19
+ gap: 16px;
20
+ background: #1a1b26;
21
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
22
+ transition: opacity 0.3s ease;
23
+ -webkit-app-region: drag; /* keep the frameless window draggable during load */
24
+ }
25
+ #boot-splash.boot-splash-hidden {
26
+ opacity: 0;
27
+ pointer-events: none;
28
+ }
29
+ .boot-spinner {
30
+ width: 34px;
31
+ height: 34px;
32
+ border-radius: 50%;
33
+ border: 3px solid rgba(122, 162, 247, 0.22);
34
+ border-top-color: #7aa2f7;
35
+ animation: boot-spin 0.8s linear infinite;
36
+ }
37
+ @keyframes boot-spin {
38
+ to { transform: rotate(360deg); }
39
+ }
40
+ .boot-splash-title {
41
+ font-size: 15px;
42
+ font-weight: 600;
43
+ letter-spacing: 0.02em;
44
+ color: #c0caf5;
45
+ }
46
+ .boot-splash-label {
47
+ font-size: 12px;
48
+ letter-spacing: 0.04em;
49
+ color: #565f89;
50
+ }
51
+ </style>
7
52
  </head>
8
53
  <body>
54
+ <div id="boot-splash">
55
+ <div class="boot-spinner"></div>
56
+ <div class="boot-splash-title">ctlsurf</div>
57
+ <div class="boot-splash-label">Starting up…</div>
58
+ </div>
9
59
  <div id="root"></div>
60
+ <script>
61
+ // Fades out and removes the boot splash. Called by the React app once its
62
+ // startup IPC has settled (see App.tsx). The timeout is a safety net so a
63
+ // hung startup never leaves the user stuck behind the splash forever.
64
+ window.__dismissSplash = function () {
65
+ var el = document.getElementById('boot-splash')
66
+ if (!el) return
67
+ el.classList.add('boot-splash-hidden')
68
+ setTimeout(function () { el.remove() }, 350)
69
+ }
70
+ setTimeout(function () { window.__dismissSplash() }, 10000)
71
+ </script>
10
72
  <script type="module" src="./main.tsx"></script>
11
73
  </body>
12
74
  </html>
@@ -6,6 +6,37 @@
6
6
  const MODEL = 'Xenova/whisper-base'
7
7
  const TARGET_SAMPLE_RATE = 16000
8
8
 
9
+ // Whisper hallucinates filler tokens ("you", "Thank you.", "Thanks for
10
+ // watching.") when fed silence. Two guards below tame that:
11
+ // - Below SILENCE_RMS we treat the clip as silent and skip the model entirely.
12
+ // - Between SILENCE_RMS and LOW_CONFIDENCE_RMS we still run the model but drop
13
+ // the result if it's *only* a known filler phrase (likely a hallucination on
14
+ // a quiet clip rather than real speech).
15
+ const SILENCE_RMS = 0.008
16
+ const LOW_CONFIDENCE_RMS = 0.02
17
+ const FILLER_PHRASES = new Set([
18
+ 'you',
19
+ 'thank you',
20
+ 'thank you very much',
21
+ 'thank you for watching',
22
+ 'thanks for watching',
23
+ 'please subscribe',
24
+ ])
25
+
26
+ // Root-mean-square amplitude of the clip — a cheap loudness proxy. Normalized
27
+ // speech sits around 0.05–0.15; a silent room is well under 0.005.
28
+ function computeRms(pcm: Float32Array): number {
29
+ if (pcm.length === 0) return 0
30
+ let sum = 0
31
+ for (let i = 0; i < pcm.length; i++) sum += pcm[i] * pcm[i]
32
+ return Math.sqrt(sum / pcm.length)
33
+ }
34
+
35
+ function isFillerOnly(text: string): boolean {
36
+ const norm = text.toLowerCase().replace(/[.!?,…"']/g, '').replace(/\s+/g, ' ').trim()
37
+ return FILLER_PHRASES.has(norm)
38
+ }
39
+
9
40
  export interface ModelProgress {
10
41
  status: string
11
42
  file?: string
@@ -76,13 +107,70 @@ async function blobToPcm16k(blob: Blob): Promise<Float32Array | null> {
76
107
  return rendered.getChannelData(0)
77
108
  }
78
109
 
79
- export async function transcribeBlob(blob: Blob, onProgress?: (p: ModelProgress) => void): Promise<string> {
80
- const transcriber = await loadTranscriber(onProgress)
110
+ export async function transcribeBlob(
111
+ blob: Blob,
112
+ onProgress?: (p: ModelProgress) => void,
113
+ // Optional live preview: called with the transcript-so-far as the model
114
+ // decodes, for progressive on-screen feedback. The return value (below) is
115
+ // still the authoritative, de-duplicated text used for the actual insert.
116
+ onPartial?: (text: string) => void,
117
+ ): Promise<string> {
118
+ // Decode first so a silent clip never triggers the (one-time, ~75MB) model
119
+ // download just to be thrown away below.
81
120
  const pcm = await blobToPcm16k(blob)
82
121
  if (!pcm) return ''
83
- const result = await transcriber(pcm)
84
- const text = Array.isArray(result)
122
+ const rms = computeRms(pcm)
123
+ if (rms < SILENCE_RMS) {
124
+ console.info(`[voice] near-silent clip (rms=${rms.toFixed(4)}); skipping transcription`)
125
+ return ''
126
+ }
127
+
128
+ const transcriber = await loadTranscriber(onProgress)
129
+
130
+ // Stream decoded text to onPartial as it's generated. `committed` holds text
131
+ // from finished chunks; `current` is the in-progress chunk — keeping them
132
+ // separate avoids the stride overlap duplicating words across 30s boundaries
133
+ // in the preview. Best-effort: if the streamer can't be built we just skip the
134
+ // preview and still return the final text.
135
+ let streamer: unknown
136
+ if (onPartial) {
137
+ try {
138
+ const { WhisperTextStreamer } = await import('@huggingface/transformers')
139
+ const tokenizer = (transcriber as unknown as {
140
+ tokenizer: ConstructorParameters<typeof WhisperTextStreamer>[0]
141
+ }).tokenizer
142
+ let committed = ''
143
+ let current = ''
144
+ streamer = new WhisperTextStreamer(tokenizer, {
145
+ on_chunk_start: () => { current = '' },
146
+ callback_function: (delta: string) => {
147
+ current += delta
148
+ onPartial((committed + current).replace(/\s+/g, ' ').trim())
149
+ },
150
+ on_chunk_end: () => { committed = (committed + current).trim() + ' '; current = '' },
151
+ })
152
+ } catch (err) {
153
+ console.warn('[voice] streaming preview unavailable', err)
154
+ }
155
+ }
156
+
157
+ // Whisper only attends to a 30s window at a time. Without chunk_length_s the
158
+ // pipeline truncates to that first window, silently dropping anything spoken
159
+ // past ~30s. Chunking transcribes the full clip in overlapping 30s windows
160
+ // (stride gives overlap so words aren't lost at the seams) so long speeches
161
+ // are captured in full.
162
+ const result = await transcriber(pcm, {
163
+ chunk_length_s: 30,
164
+ stride_length_s: 5,
165
+ ...(streamer ? { streamer } : {}),
166
+ })
167
+ const text = (Array.isArray(result)
85
168
  ? result.map((r) => r.text).join(' ')
86
- : result?.text
87
- return (text || '').trim()
169
+ : result?.text || '').trim()
170
+
171
+ if (text && rms < LOW_CONFIDENCE_RMS && isFillerOnly(text)) {
172
+ console.info(`[voice] dropping filler-only output "${text}" from quiet clip (rms=${rms.toFixed(4)})`)
173
+ return ''
174
+ }
175
+ return text
88
176
  }