@phenx-inc/ctlsurf 0.8.0 → 0.9.0

package/src/renderer/components/FloatingMic.tsx

@@ -1,5 +1,6 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
 import { VoiceInput } from './VoiceInput'
+import { speech } from '../lib/speech'
 
 // A draggable, dismissable push-to-talk mic that floats over the panes. It wraps
 // the same <VoiceInput> push-to-talk logic used in the titlebar; only the chrome
@@ -36,6 +37,29 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
   // Pointer-to-element offset captured at drag start; null when not dragging.
   const dragRef = useRef<{ dx: number; dy: number } | null>(null)
 
+  // Read-aloud (TTS) of the last agent reply. `speaking` mirrors the shared
+  // speech singleton; `note` is a transient message (e.g. nothing to read yet).
+  const [speaking, setSpeaking] = useState(false)
+  const [note, setNote] = useState<string | null>(null)
+  useEffect(() => speech.onActivity(setSpeaking), [])
+  useEffect(() => {
+    if (!note) return
+    const t = setTimeout(() => setNote(null), 3000)
+    return () => clearTimeout(t)
+  }, [note])
+
+  const onReadAloud = useCallback(async () => {
+    // Toggle: a second click while speaking stops it.
+    if (speaking) { speech.stop(); return }
+    // Prime the audio context from this click (synchronously, before the await)
+    // so neural playback isn't blocked by the autoplay policy.
+    speech.unlock()
+    let text: string | null = null
+    try { text = await window.worker.getLastReply() } catch { /* ignore */ }
+    if (text) speech.enqueue(text)
+    else setNote('No reply to read yet')
+  }, [speaking])
+
   // Keep the button fully inside the viewport (used on drag, mount, and resize).
   const clamp = useCallback((x: number, y: number): Pos => {
     const el = elRef.current
@@ -123,6 +147,30 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
         </button>
       </div>
       <VoiceInput variant="floating" onTranscript={onTranscript} />
+      <button
+        type="button"
+        className={`floating-mic-read ${speaking ? 'speaking' : ''}`}
+        // Don't let a click on the button start a drag.
+        onPointerDown={(e) => e.stopPropagation()}
+        onClick={onReadAloud}
+        title={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
+        aria-label={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
+        aria-pressed={speaking}
+      >
+        {speaking ? (
+          <svg viewBox="0 0 24 24" width="14" height="14" fill="currentColor" aria-hidden="true">
+            <rect x="6" y="6" width="12" height="12" rx="2" />
+          </svg>
+        ) : (
+          <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor"
+            strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+            <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5" />
+            <path d="M15.54 8.46a5 5 0 0 1 0 7.07" />
+            <path d="M19.07 4.93a10 10 0 0 1 0 14.14" />
+          </svg>
+        )}
+      </button>
+      {note && <div className="floating-mic-note">{note}</div>}
     </div>
   )
 }

package/src/renderer/components/VoiceInput.tsx

@@ -1,76 +1,26 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
 import { transcribeBlob, type ModelProgress } from '../lib/localWhisper'
 
-// ─── Minimal Web Speech API typings ──────────────────
-// webkitSpeechRecognition isn't in the standard DOM lib, so declare just the
-// surface we use. This API is frequently unavailable inside Electron (Chromium
-// ships without Google's speech backend); when it fails we fall back to a local
-// Whisper model (see ../lib/localWhisper).
-
-interface SpeechRecognitionResult {
-  isFinal: boolean
-  0: { transcript: string }
-}
-interface SpeechRecognitionEvent {
-  resultIndex: number
-  results: { length: number;[index: number]: SpeechRecognitionResult }
-}
-interface SpeechRecognitionErrorEvent { error: string }
-interface SpeechRecognitionLike {
-  lang: string
-  continuous: boolean
-  interimResults: boolean
-  start: () => void
-  stop: () => void
-  abort: () => void
-  onresult: ((e: SpeechRecognitionEvent) => void) | null
-  onerror: ((e: SpeechRecognitionErrorEvent) => void) | null
-  onend: (() => void) | null
-}
-type SpeechRecognitionCtor = new () => SpeechRecognitionLike
+// ─── Capabilities ────────────────────────────────────
+// Speech is transcribed entirely on-device via a local Whisper model (see
+// ../lib/localWhisper). The browser's Web Speech API is intentionally not used:
+// it's a cloud service whose Google backend is unreachable inside Electron
+// (Chromium ships without the API key), so it never works in this app.
 
-function getRecognitionCtor(): SpeechRecognitionCtor | null {
-  const w = window as unknown as {
-    SpeechRecognition?: SpeechRecognitionCtor
-    webkitSpeechRecognition?: SpeechRecognitionCtor
-  }
-  return w.SpeechRecognition || w.webkitSpeechRecognition || null
-}
-
-// ─── Capabilities & engine selection ─────────────────
-
-type Engine = 'web-speech' | 'local'
 type Phase = 'idle' | 'listening' | 'transcribing'
 
-const ENGINE_KEY = 'ctlsurf.voiceEngine'
-// Persisted audioinput deviceId for the local engine; null/absent = OS default.
+// Persisted audioinput deviceId; null/absent = OS default.
 const DEVICE_KEY = 'ctlsurf.voiceDeviceId'
 
 function loadDeviceId(): string | null {
   try { return localStorage.getItem(DEVICE_KEY) || null } catch { return null }
 }
 
-const WEB_SPEECH_SUPPORTED = getRecognitionCtor() !== null
 const LOCAL_SUPPORTED =
   typeof navigator !== 'undefined' &&
   !!navigator.mediaDevices?.getUserMedia &&
   typeof MediaRecorder !== 'undefined' &&
   typeof OfflineAudioContext !== 'undefined'
-const ANY_SUPPORTED = WEB_SPEECH_SUPPORTED || LOCAL_SUPPORTED
-
-function loadInitialEngine(): Engine {
-  if (!WEB_SPEECH_SUPPORTED && LOCAL_SUPPORTED) return 'local'
-  try {
-    if (localStorage.getItem(ENGINE_KEY) === 'local' && LOCAL_SUPPORTED) return 'local'
-  } catch { /* ignore */ }
-  return WEB_SPEECH_SUPPORTED ? 'web-speech' : 'local'
-}
-
-// Web Speech errors that mean the engine itself is unreachable (vs. a mic
-// permission/hardware problem, which would also break the local fallback).
-function isEngineUnavailable(code: string): boolean {
-  return code === 'network' || code === 'service-not-allowed'
-}
 
 function describeMicError(err: unknown): string {
   const name = (err as { name?: string })?.name
@@ -88,14 +38,14 @@ interface VoiceInputProps {
 }
 
 export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputProps) {
-  const [engine, setEngine] = useState<Engine>(loadInitialEngine)
   const [phase, setPhase] = useState<Phase>('idle')
-  const [interim, setInterim] = useState('')
   const [modelPct, setModelPct] = useState<number | null>(null)
+  // Transcript-so-far streamed from the model during transcription (live preview).
+  const [partial, setPartial] = useState('')
   const [error, setError] = useState<string | null>(null)
   const [notice, setNotice] = useState<string | null>(null)
 
-  // Mic source selection (local engine only)
+  // Mic source selection
   const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
   const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(loadDeviceId)
   const [showDevicePicker, setShowDevicePicker] = useState(false)
@@ -103,19 +53,13 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
   useEffect(() => { selectedDeviceIdRef.current = selectedDeviceId }, [selectedDeviceId])
   const wrapRef = useRef<HTMLDivElement>(null)
 
-  // Web Speech refs
-  const recognitionRef = useRef<SpeechRecognitionLike | null>(null)
-  const finalRef = useRef('')
-  // Local (Whisper) refs
+  // Recording refs
   const streamRef = useRef<MediaStream | null>(null)
   const recorderRef = useRef<MediaRecorder | null>(null)
   const chunksRef = useRef<Blob[]>([])
   // Set true when the user releases before getUserMedia resolves (quick tap).
   const cancelGestureRef = useRef(false)
 
-  const engineRef = useRef(engine)
-  useEffect(() => { engineRef.current = engine }, [engine])
-
   const onTranscriptRef = useRef(onTranscript)
   useEffect(() => { onTranscriptRef.current = onTranscript }, [onTranscript])
 
@@ -131,12 +75,6 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
     return () => clearTimeout(t)
   }, [notice])
 
-  const switchToLocal = useCallback((reason: string) => {
-    try { localStorage.setItem(ENGINE_KEY, 'local') } catch { /* ignore */ }
-    setEngine('local')
-    setNotice(reason)
-  }, [])
-
   const stopStream = useCallback(() => {
     streamRef.current?.getTracks().forEach((t) => t.stop())
     streamRef.current = null
@@ -224,66 +162,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
     return () => document.removeEventListener('mousedown', onDocDown)
   }, [showDevicePicker])
 
-  // ─── Web Speech engine ─────────────────────────────
-
-  const startWebSpeech = useCallback(() => {
-    const Ctor = getRecognitionCtor()
-    if (!Ctor || recognitionRef.current) return
-    setError(null); setNotice(null); setInterim('')
-    finalRef.current = ''
-
-    const rec = new Ctor()
-    rec.lang = navigator.language || 'en-US'
-    rec.continuous = true
-    rec.interimResults = true
-
-    rec.onresult = (event) => {
-      let finalText = ''
-      let interimText = ''
-      for (let i = 0; i < event.results.length; i++) {
-        const res = event.results[i]
-        if (res.isFinal) finalText += res[0].transcript
-        else interimText += res[0].transcript
-      }
-      finalRef.current = finalText
-      setInterim(interimText)
-    }
-
-    rec.onerror = (event) => {
-      if (isEngineUnavailable(event.error) && LOCAL_SUPPORTED) {
-        // The streamed audio is gone; switch engines and ask for a retry.
-        finalRef.current = ''
-        switchToLocal('Voice service unavailable — switched to on-device. Press again.')
-      } else if (event.error !== 'no-speech' && event.error !== 'aborted') {
-        setError(event.error === 'not-allowed' ? 'Microphone access denied' : `Voice error: ${event.error}`)
-      }
-    }
-
-    rec.onend = () => {
-      const text = finalRef.current.trim()
-      recognitionRef.current = null
-      setPhase('idle')
-      setInterim('')
-      if (text) onTranscriptRef.current(text)
-    }
-
-    recognitionRef.current = rec
-    try {
-      rec.start()
-      setPhase('listening')
-    } catch (err) {
-      recognitionRef.current = null
-      setPhase('idle')
-      setError('Could not start microphone')
-      console.error('[voice] web speech start failed', err)
-    }
-  }, [switchToLocal])
-
-  const stopWebSpeech = useCallback(() => {
-    try { recognitionRef.current?.stop() } catch { /* already stopped */ }
-  }, [])
-
-  // ─── Local (Whisper) engine ────────────────────────
+  // ─── On-device (Whisper) transcription ─────────────
 
   const handleModelProgress = useCallback((p: ModelProgress) => {
     if (p.status === 'progress' && typeof p.progress === 'number') {
@@ -299,9 +178,14 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
     if (blob.size === 0) { setPhase('idle'); return }
 
     setPhase('transcribing')
-    setInterim('')
+    setPartial('')
     try {
-      const text = await transcribeBlob(blob, handleModelProgress)
+      const text = await transcribeBlob(blob, handleModelProgress, (t) => {
+        // First streamed token means the model is loaded and decoding — drop the
+        // download indicator and start showing the live transcript.
+        setModelPct(null)
+        setPartial(t)
+      })
       if (text) onTranscriptRef.current(text)
       else setNotice('No speech detected — check the mic source (▾).')
     } catch (err) {
@@ -310,11 +194,12 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
     } finally {
       setPhase('idle')
       setModelPct(null)
+      setPartial('')
     }
   }, [stopStream, handleModelProgress])
 
   const startLocal = useCallback(async () => {
-    setError(null); setNotice(null); setInterim('')
+    setError(null); setNotice(null)
     cancelGestureRef.current = false
     try {
       const stream = await getStream()
@@ -351,21 +236,18 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
   // ─── Push-to-talk gesture ──────────────────────────
 
   const handlePointerDown = (e: React.PointerEvent) => {
-    if (!ANY_SUPPORTED || phase !== 'idle') return
+    if (!LOCAL_SUPPORTED || phase !== 'idle') return
     e.preventDefault()
     e.currentTarget.setPointerCapture?.(e.pointerId)
-    if (engineRef.current === 'web-speech' && WEB_SPEECH_SUPPORTED) startWebSpeech()
-    else if (LOCAL_SUPPORTED) void startLocal()
+    void startLocal()
   }
   const handlePointerUp = (e: React.PointerEvent) => {
     e.currentTarget.releasePointerCapture?.(e.pointerId)
-    if (engineRef.current === 'web-speech') stopWebSpeech()
-    else stopLocal()
+    stopLocal()
   }
 
   // Clean up on unmount.
   useEffect(() => () => {
-    try { recognitionRef.current?.abort() } catch { /* ignore */ }
     try { recorderRef.current?.stop() } catch { /* ignore */ }
     streamRef.current?.getTracks().forEach((t) => t.stop())
   }, [])
@@ -375,21 +257,27 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
   const listening = phase === 'listening'
   const busy = phase === 'transcribing'
 
-  const title = !ANY_SUPPORTED
+  const title = !LOCAL_SUPPORTED
     ? 'Voice typing not supported in this build'
     : listening
       ? 'Listening… release to insert'
       : busy
         ? 'Transcribing…'
-        : engine === 'local'
-          ? 'Hold to talk (on-device) — speech is typed into the terminal'
-          : 'Hold to talk — speech is typed into the terminal'
+        : 'Hold to talk (on-device) — speech is typed into the terminal'
 
   let chip: { kind: 'listening' | 'busy' | 'notice' | 'error'; text: string } | null = null
   if (error && phase === 'idle') chip = { kind: 'error', text: error }
   else if (notice && phase === 'idle') chip = { kind: 'notice', text: notice }
-  else if (listening) chip = { kind: 'listening', text: interim || (engine === 'local' ? 'Recording…' : 'Listening…') }
-  else if (busy) chip = { kind: 'busy', text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : 'Transcribing…' }
+  else if (listening) chip = { kind: 'listening', text: 'Recording…' }
+  else if (busy) {
+    // Show the most recent ~90 chars of the live transcript as a ticker so a long
+    // dictation doesn't overflow the chip.
+    const preview = partial.length > 90 ? '…' + partial.slice(-90) : partial
+    chip = {
+      kind: 'busy',
+      text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : (preview || 'Transcribing…'),
+    }
+  }
 
   const floating = variant === 'floating'
   const btnClass = floating
@@ -408,7 +296,6 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
   const activeLabel = selectedDeviceId
     ? (realDevices.find((d) => d.deviceId === selectedDeviceId)?.label || 'Selected microphone')
     : defaultLabel
-  // Source selection only affects the local (getUserMedia) engine.
   const showSourcePicker = floating && LOCAL_SUPPORTED
 
   return (
@@ -416,7 +303,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
       <button
         type="button"
         className={btnClass}
-        disabled={!ANY_SUPPORTED}
+        disabled={!LOCAL_SUPPORTED}
         onPointerDown={handlePointerDown}
         onPointerUp={handlePointerUp}
         onPointerCancel={handlePointerUp}

package/src/renderer/index.html

@@ -4,9 +4,71 @@
   <meta charset="UTF-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>ctlsurf-worker</title>
+  <!-- Boot splash styles are inlined here so they paint on the first frame,
+       before the JS bundle and styles.css have loaded. Without this the window
+       shows an empty dark pane while React mounts and the startup IPC runs. -->
+  <style>
+    #boot-splash {
+      position: fixed;
+      inset: 0;
+      z-index: 9999;
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      justify-content: center;
+      gap: 16px;
+      background: #1a1b26;
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+      transition: opacity 0.3s ease;
+      -webkit-app-region: drag; /* keep the frameless window draggable during load */
+    }
+    #boot-splash.boot-splash-hidden {
+      opacity: 0;
+      pointer-events: none;
+    }
+    .boot-spinner {
+      width: 34px;
+      height: 34px;
+      border-radius: 50%;
+      border: 3px solid rgba(122, 162, 247, 0.22);
+      border-top-color: #7aa2f7;
+      animation: boot-spin 0.8s linear infinite;
+    }
+    @keyframes boot-spin {
+      to { transform: rotate(360deg); }
+    }
+    .boot-splash-title {
+      font-size: 15px;
+      font-weight: 600;
+      letter-spacing: 0.02em;
+      color: #c0caf5;
+    }
+    .boot-splash-label {
+      font-size: 12px;
+      letter-spacing: 0.04em;
+      color: #565f89;
+    }
+  </style>
 </head>
 <body>
+  <div id="boot-splash">
+    <div class="boot-spinner"></div>
+    <div class="boot-splash-title">ctlsurf</div>
+    <div class="boot-splash-label">Starting up…</div>
+  </div>
   <div id="root"></div>
+  <script>
+    // Fades out and removes the boot splash. Called by the React app once its
+    // startup IPC has settled (see App.tsx). The timeout is a safety net so a
+    // hung startup never leaves the user stuck behind the splash forever.
+    window.__dismissSplash = function () {
+      var el = document.getElementById('boot-splash')
+      if (!el) return
+      el.classList.add('boot-splash-hidden')
+      setTimeout(function () { el.remove() }, 350)
+    }
+    setTimeout(function () { window.__dismissSplash() }, 10000)
+  </script>
   <script type="module" src="./main.tsx"></script>
 </body>
 </html>

package/src/renderer/lib/localWhisper.ts

@@ -107,7 +107,14 @@ async function blobToPcm16k(blob: Blob): Promise<Float32Array | null> {
   return rendered.getChannelData(0)
 }
 
-export async function transcribeBlob(blob: Blob, onProgress?: (p: ModelProgress) => void): Promise<string> {
+export async function transcribeBlob(
+  blob: Blob,
+  onProgress?: (p: ModelProgress) => void,
+  // Optional live preview: called with the transcript-so-far as the model
+  // decodes, for progressive on-screen feedback. The return value (below) is
+  // still the authoritative, de-duplicated text used for the actual insert.
+  onPartial?: (text: string) => void,
+): Promise<string> {
   // Decode first so a silent clip never triggers the (one-time, ~75MB) model
   // download just to be thrown away below.
   const pcm = await blobToPcm16k(blob)
@@ -119,7 +126,44 @@ export async function transcribeBlob(blob: Blob, onProgress?: (p: ModelProgress)
   }
 
   const transcriber = await loadTranscriber(onProgress)
-  const result = await transcriber(pcm)
+
+  // Stream decoded text to onPartial as it's generated. `committed` holds text
+  // from finished chunks; `current` is the in-progress chunk — keeping them
+  // separate avoids the stride overlap duplicating words across 30s boundaries
+  // in the preview. Best-effort: if the streamer can't be built we just skip the
+  // preview and still return the final text.
+  let streamer: unknown
+  if (onPartial) {
+    try {
+      const { WhisperTextStreamer } = await import('@huggingface/transformers')
+      const tokenizer = (transcriber as unknown as {
+        tokenizer: ConstructorParameters<typeof WhisperTextStreamer>[0]
+      }).tokenizer
+      let committed = ''
+      let current = ''
+      streamer = new WhisperTextStreamer(tokenizer, {
+        on_chunk_start: () => { current = '' },
+        callback_function: (delta: string) => {
+          current += delta
+          onPartial((committed + current).replace(/\s+/g, ' ').trim())
+        },
+        on_chunk_end: () => { committed = (committed + current).trim() + ' '; current = '' },
+      })
+    } catch (err) {
+      console.warn('[voice] streaming preview unavailable', err)
+    }
+  }
+
+  // Whisper only attends to a 30s window at a time. Without chunk_length_s the
+  // pipeline truncates to that first window, silently dropping anything spoken
+  // past ~30s. Chunking transcribes the full clip in overlapping 30s windows
+  // (stride gives overlap so words aren't lost at the seams) so long speeches
+  // are captured in full.
+  const result = await transcriber(pcm, {
+    chunk_length_s: 30,
+    stride_length_s: 5,
+    ...(streamer ? { streamer } : {}),
+  })
   const text = (Array.isArray(result)
     ? result.map((r) => r.text).join(' ')
     : result?.text || '').trim()

package/src/renderer/lib/speech.ts

@@ -164,6 +164,23 @@ class SpeechController {
   onActivityChange: ((active: boolean) => void) | null = null
   onError: ((message: string) => void) | null = null
 
+  // Multiple UI surfaces (titlebar SpeakControls + floating read button) need
+  // the speaking state at once, so beyond the single onActivityChange slot we
+  // also keep a listener set.
+  private activityListeners = new Set<(active: boolean) => void>()
+
+  get isActive(): boolean {
+    return this.active
+  }
+
+  // Subscribe to speaking-state changes. Fires immediately with the current
+  // state so a late subscriber syncs. Returns an unsubscribe function.
+  onActivity(fn: (active: boolean) => void): () => void {
+    this.activityListeners.add(fn)
+    fn(this.active)
+    return () => { this.activityListeners.delete(fn) }
+  }
+
   // Prime/resume the AudioContext from a user gesture so neural playback isn't
   // blocked by the browser autoplay policy (the System engine is unaffected).
   unlock(): void {
@@ -216,6 +233,7 @@ class SpeechController {
     if (this.active === a) return
     this.active = a
     this.onActivityChange?.(a)
+    this.activityListeners.forEach((fn) => fn(a))
   }
 
   private async drain(): Promise<void> {

package/src/renderer/styles.css

@@ -701,6 +701,40 @@ html, body, #root {
 }
 .floating-mic-hide:hover { color: #f7768e; }
 
+/* Smaller sibling of the round mic: read the last agent reply aloud. */
+.floating-mic-read {
+  width: 30px;
+  height: 30px;
+  border-radius: 50%;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  background: #2a2b3d;
+  border: 1px solid #3b3d57;
+  color: #a9b1d6;
+  cursor: pointer;
+  transition: all 0.15s;
+}
+.floating-mic-read:hover { border-color: #565f89; color: #c0caf5; }
+.floating-mic-read.speaking {
+  color: #7aa2f7;
+  border-color: #7aa2f7;
+}
+.floating-mic-note {
+  position: absolute;
+  bottom: -22px;
+  left: 50%;
+  transform: translateX(-50%);
+  white-space: nowrap;
+  font-size: 11px;
+  color: #565f89;
+  background: #16161e;
+  border: 1px solid #3b3d57;
+  border-radius: 6px;
+  padding: 2px 6px;
+  pointer-events: none;
+}
+
 .voice-btn-floating {
   width: 48px;
   height: 48px;