@phenx-inc/ctlsurf 0.7.0 → 0.9.0

package/src/preload/index.ts

@@ -109,6 +109,20 @@ const api = {
   setLogChat: (enabled: boolean): Promise<{ enabled: boolean }> =>
     ipcRenderer.invoke('logchat:set', enabled),
 
+  // Speak agent replies (global, Electron-only)
+  getSpeakReplies: (): Promise<{ enabled: boolean }> =>
+    ipcRenderer.invoke('speak:get'),
+  setSpeakReplies: (enabled: boolean): Promise<{ enabled: boolean }> =>
+    ipcRenderer.invoke('speak:set', enabled),
+  onAgentMessage: (callback: (text: string) => void) => {
+    const listener = (_event: Electron.IpcRendererEvent, text: string) => callback(text)
+    ipcRenderer.on('agent:message', listener)
+    return () => ipcRenderer.removeListener('agent:message', listener)
+  },
+  // Latest assistant reply from the active tab's transcript (null if none).
+  getLastReply: (): Promise<string | null> =>
+    ipcRenderer.invoke('agent:getLastReply'),
+
   // Filesystem
   readDir: (dirPath: string): Promise<Array<{ name: string; path: string; isDirectory: boolean }>> =>
     ipcRenderer.invoke('fs:readDir', dirPath),

package/src/renderer/App.tsx

@@ -1,6 +1,7 @@
 import { useState, useEffect, useCallback, useRef } from 'react'
 import { TerminalPanel, destroyTerminal, focusTerminal } from './components/TerminalPanel'
 import { FloatingMic } from './components/FloatingMic'
+import { SpeakControls } from './components/SpeakControls'
 import { CtlsurfPanel } from './components/CtlsurfPanel'
 import { EditorPanel } from './components/EditorPanel'
 import { AgentPicker } from './components/AgentPicker'
@@ -56,6 +57,10 @@ declare global {
       setTracking: (enabled: boolean) => Promise<{ active: boolean }>
       getLogChat: () => Promise<{ enabled: boolean }>
       setLogChat: (enabled: boolean) => Promise<{ enabled: boolean }>
+      getSpeakReplies: () => Promise<{ enabled: boolean }>
+      setSpeakReplies: (enabled: boolean) => Promise<{ enabled: boolean }>
+      onAgentMessage: (callback: (text: string) => void) => () => void
+      getLastReply: () => Promise<string | null>
       createProject: () => Promise<{ ok: boolean; folder_id?: string; error?: string }>
       getWebviewInfo: () => Promise<{
         frontendUrl: string; pageUrl?: string; authenticated: boolean;
@@ -151,16 +156,22 @@ export default function App() {
 
   useEffect(() => {
     async function init() {
-      const list = await window.worker.listAgents()
-      setAgents(list)
-      const status = await window.worker.getWorkerStatus()
-      setWsStatus(status)
-      // Load initial cwd for the picker
-      const initialCwd = await window.worker.getCwd().catch(() => window.worker.getHomePath())
-      setCwd(initialCwd)
-      cwdRef.current = initialCwd
-      // Connected ctlsurf project name for the header (null until a worker registers)
-      window.worker.getProjectName().then(setProjectName).catch(() => { /* ignore */ })
+      try {
+        const list = await window.worker.listAgents()
+        setAgents(list)
+        const status = await window.worker.getWorkerStatus()
+        setWsStatus(status)
+        // Load initial cwd for the picker
+        const initialCwd = await window.worker.getCwd().catch(() => window.worker.getHomePath())
+        setCwd(initialCwd)
+        cwdRef.current = initialCwd
+        // Connected ctlsurf project name for the header (null until a worker registers)
+        window.worker.getProjectName().then(setProjectName).catch(() => { /* ignore */ })
+      } finally {
+        // Dismiss the boot splash painted by index.html now that startup IPC has
+        // settled. In `finally` so a failed IPC call never leaves it stuck on.
+        ;(window as unknown as { __dismissSplash?: () => void }).__dismissSplash?.()
+      }
     }
     init()
   }, [])
@@ -221,8 +232,15 @@ export default function App() {
   const handleVoiceTranscript = useCallback((text: string) => {
     const trimmed = text.trim()
     if (!trimmed) return
-    window.worker.writePty(activeTabId, trimmed + '\r')
+    // Send the text and the submit Enter as TWO separate writes. A long transcript
+    // glued to its trailing \r arrives as one fast burst, which the agent's input
+    // box treats as a paste and turns the carriage return into a literal newline
+    // instead of submitting. Delaying the Enter lets the text be consumed first so
+    // \r registers as a real submit keystroke. (80ms is the tunable knob if very
+    // long input ever needs more time to drain.)
+    window.worker.writePty(activeTabId, trimmed)
     focusTerminal(activeTabId)
+    setTimeout(() => window.worker.writePty(activeTabId, '\r'), 80)
   }, [activeTabId])
 
   const cwdRef = useRef<string | null>(null)
@@ -504,6 +522,7 @@ export default function App() {
               <line x1="8" y1="23" x2="16" y2="23" />
             </svg>
           </button>
+          <SpeakControls />
           <span className="titlebar-separator" />
           {agents.map(a => {
             const activeTab = tabs.find(t => t.id === activeTabId)

package/src/renderer/components/FloatingMic.tsx

@@ -1,5 +1,6 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
 import { VoiceInput } from './VoiceInput'
+import { speech } from '../lib/speech'
 
 // A draggable, dismissable push-to-talk mic that floats over the panes. It wraps
 // the same <VoiceInput> push-to-talk logic used in the titlebar; only the chrome
@@ -36,6 +37,29 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
   // Pointer-to-element offset captured at drag start; null when not dragging.
   const dragRef = useRef<{ dx: number; dy: number } | null>(null)
 
+  // Read-aloud (TTS) of the last agent reply. `speaking` mirrors the shared
+  // speech singleton; `note` is a transient message (e.g. nothing to read yet).
+  const [speaking, setSpeaking] = useState(false)
+  const [note, setNote] = useState<string | null>(null)
+  useEffect(() => speech.onActivity(setSpeaking), [])
+  useEffect(() => {
+    if (!note) return
+    const t = setTimeout(() => setNote(null), 3000)
+    return () => clearTimeout(t)
+  }, [note])
+
+  const onReadAloud = useCallback(async () => {
+    // Toggle: a second click while speaking stops it.
+    if (speaking) { speech.stop(); return }
+    // Prime the audio context from this click (synchronously, before the await)
+    // so neural playback isn't blocked by the autoplay policy.
+    speech.unlock()
+    let text: string | null = null
+    try { text = await window.worker.getLastReply() } catch { /* ignore */ }
+    if (text) speech.enqueue(text)
+    else setNote('No reply to read yet')
+  }, [speaking])
+
   // Keep the button fully inside the viewport (used on drag, mount, and resize).
   const clamp = useCallback((x: number, y: number): Pos => {
     const el = elRef.current
@@ -123,6 +147,30 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
         </button>
       </div>
       <VoiceInput variant="floating" onTranscript={onTranscript} />
+      <button
+        type="button"
+        className={`floating-mic-read ${speaking ? 'speaking' : ''}`}
+        // Don't let a click on the button start a drag.
+        onPointerDown={(e) => e.stopPropagation()}
+        onClick={onReadAloud}
+        title={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
+        aria-label={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
+        aria-pressed={speaking}
+      >
+        {speaking ? (
+          <svg viewBox="0 0 24 24" width="14" height="14" fill="currentColor" aria-hidden="true">
+            <rect x="6" y="6" width="12" height="12" rx="2" />
+          </svg>
+        ) : (
+          <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor"
+            strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+            <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5" />
+            <path d="M15.54 8.46a5 5 0 0 1 0 7.07" />
+            <path d="M19.07 4.93a10 10 0 0 1 0 14.14" />
+          </svg>
+        )}
+      </button>
+      {note && <div className="floating-mic-note">{note}</div>}
     </div>
   )
 }

package/src/renderer/components/SpeakControls.tsx

@@ -0,0 +1,235 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import {
+  speech,
+  getEngine,
+  setEngine as persistEngine,
+  getVoiceURI,
+  setVoiceURI as persistVoiceURI,
+  getRate,
+  setRate as persistRate,
+  listWebVoices,
+  type TtsEngineId,
+  type TtsModelProgress,
+} from '../lib/speech'
+
+// Titlebar control for "speak agent replies" (Electron desktop only). A speaker
+// toggle drives the main-process speakReplies setting (which runs the transcript
+// tailer); a ▾ dropdown picks the engine, voice, and rate.
+
+const SAMPLE = 'This is how spoken agent replies will sound.'
+
+export function SpeakControls() {
+  const [enabled, setEnabled] = useState(false)
+  const [engine, setEngineState] = useState<TtsEngineId>(getEngine)
+  const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([])
+  const [voiceURI, setVoiceURIState] = useState<string | null>(getVoiceURI)
+  const [rate, setRateState] = useState<number>(getRate)
+  const [showMenu, setShowMenu] = useState(false)
+  const [modelPct, setModelPct] = useState<number | null>(null)
+  const [speaking, setSpeaking] = useState(false)
+  const [error, setError] = useState<string | null>(null)
+  const wrapRef = useRef<HTMLDivElement>(null)
+
+  // Reflect the persisted main-process setting on mount.
+  useEffect(() => {
+    let alive = true
+    window.worker.getSpeakReplies().then((r) => { if (alive) setEnabled(!!r.enabled) }).catch(() => {})
+    return () => { alive = false }
+  }, [])
+
+  // Speak replies as they arrive. Main only forwards when the setting is on, so
+  // this is a no-op while disabled.
+  useEffect(() => {
+    const off = window.worker.onAgentMessage((text) => speech.enqueue(text))
+    return off
+  }, [])
+
+  // Surface neural-model download/transcribe progress on the button.
+  useEffect(() => {
+    speech.onModelProgress = (p: TtsModelProgress | null) => {
+      if (p && p.status === 'progress' && typeof p.progress === 'number') {
+        setModelPct(Math.min(100, Math.round(p.progress)))
+      } else if (!p) {
+        setModelPct(null)
+      }
+    }
+    speech.onActivityChange = (a) => setSpeaking(a)
+    speech.onError = (msg) => setError(msg)
+    return () => {
+      speech.onModelProgress = null
+      speech.onActivityChange = null
+      speech.onError = null
+    }
+  }, [])
+
+  // Auto-dismiss the error chip.
+  useEffect(() => {
+    if (!error) return
+    const t = setTimeout(() => setError(null), 4000)
+    return () => clearTimeout(t)
+  }, [error])
+
+  // Web Speech voices populate asynchronously.
+  useEffect(() => {
+    const load = () => setVoices(listWebVoices())
+    load()
+    if (typeof speechSynthesis !== 'undefined') {
+      speechSynthesis.addEventListener('voiceschanged', load)
+      return () => speechSynthesis.removeEventListener('voiceschanged', load)
+    }
+  }, [])
+
+  // Close the menu on an outside click.
+  useEffect(() => {
+    if (!showMenu) return
+    const onDown = (e: MouseEvent) => {
+      if (!wrapRef.current?.contains(e.target as Node)) setShowMenu(false)
+    }
+    document.addEventListener('mousedown', onDown)
+    return () => document.removeEventListener('mousedown', onDown)
+  }, [showMenu])
+
+  const toggle = useCallback(async () => {
+    const next = !enabled
+    setEnabled(next)
+    // Prime the audio context from this click so neural playback isn't blocked
+    // by autoplay policy when a reply later arrives without a gesture.
+    if (next) speech.unlock()
+    else speech.stop()
+    try { await window.worker.setSpeakReplies(next) } catch { /* ignore */ }
+  }, [enabled])
+
+  const chooseEngine = useCallback((id: TtsEngineId) => {
+    persistEngine(id)
+    setEngineState(id)
+    speech.stop()
+    // Preload the neural model so the first reply isn't slowed by download.
+    if (id === 'neural') speech.warmup()
+  }, [])
+
+  const chooseVoice = useCallback((uri: string) => {
+    const v = uri || null
+    persistVoiceURI(v)
+    setVoiceURIState(v)
+  }, [])
+
+  const changeRate = useCallback((r: number) => {
+    persistRate(r)
+    setRateState(r)
+  }, [])
+
+  const testVoice = useCallback(() => {
+    speech.unlock()
+    speech.stop()
+    speech.enqueue(SAMPLE)
+  }, [])
+
+  const title = enabled ? 'Spoken replies on — click to mute' : 'Speak agent replies (off)'
+
+  return (
+    <div className="speak-controls" ref={wrapRef}>
+      <button
+        type="button"
+        className={`titlebar-btn titlebar-icon-btn speak-btn ${enabled ? 'active' : ''}`}
+        onClick={toggle}
+        title={title}
+        aria-label="Toggle spoken agent replies"
+        aria-pressed={enabled}
+      >
+        <svg viewBox="0 0 24 24" width="13" height="13" fill="none" stroke="currentColor"
+          strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+          <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5" />
+          {enabled ? (
+            <>
+              <path d="M15.54 8.46a5 5 0 0 1 0 7.07" />
+              <path d="M19.07 4.93a10 10 0 0 1 0 14.14" />
+            </>
+          ) : (
+            <line x1="23" y1="9" x2="17" y2="15" />
+          )}
+        </svg>
+        {modelPct !== null && <span className="speak-pct">{modelPct}%</span>}
+      </button>
+      {speaking && (
+        <button
+          type="button"
+          className="titlebar-btn titlebar-icon-btn speak-stop"
+          onClick={() => speech.stop()}
+          title="Stop speaking"
+          aria-label="Stop speaking"
+        >
+          <svg viewBox="0 0 24 24" width="11" height="11" fill="currentColor" aria-hidden="true">
+            <rect x="5" y="5" width="14" height="14" rx="2" />
+          </svg>
+        </button>
+      )}
+      <button
+        type="button"
+        className="titlebar-btn speak-caret"
+        onClick={() => setShowMenu((v) => !v)}
+        title="Voice options"
+        aria-label="Voice options"
+        aria-expanded={showMenu}
+      >
+        ▾
+      </button>
+
+      {showMenu && (
+        <div className="speak-menu" role="menu">
+          <div className="speak-menu-head">Engine</div>
+          <button
+            type="button"
+            className={`speak-menu-item ${engine === 'web' ? 'active' : ''}`}
+            onClick={() => chooseEngine('web')}
+          >
+            <span className="speak-menu-check">{engine === 'web' ? '✓' : ''}</span>
+            <span>System voice (instant)</span>
+          </button>
+          <button
+            type="button"
+            className={`speak-menu-item ${engine === 'neural' ? 'active' : ''}`}
+            onClick={() => chooseEngine('neural')}
+          >
+            <span className="speak-menu-check">{engine === 'neural' ? '✓' : ''}</span>
+            <span>Neural voice (downloads)</span>
+          </button>
+
+          {engine === 'web' && (
+            <>
+              <div className="speak-menu-head">Voice</div>
+              <select
+                className="speak-select"
+                value={voiceURI || ''}
+                onChange={(e) => chooseVoice(e.target.value)}
+              >
+                <option value="">System default</option>
+                {voices.map((v) => (
+                  <option key={v.voiceURI} value={v.voiceURI}>
+                    {v.name} {v.lang ? `(${v.lang})` : ''}
+                  </option>
+                ))}
+              </select>
+            </>
+          )}
+
+          <div className="speak-menu-head">Rate · {rate.toFixed(1)}×</div>
+          <input
+            className="speak-rate"
+            type="range"
+            min={0.5}
+            max={2}
+            step={0.1}
+            value={rate}
+            onChange={(e) => changeRate(Number(e.target.value))}
+          />
+
+          <div className="speak-menu-row">
+            <button type="button" className="speak-menu-btn" onClick={testVoice}>Test</button>
+            <button type="button" className="speak-menu-btn" onClick={() => speech.stop()}>Stop</button>
+          </div>
+        </div>
+      )}
+      {error && <div className="speak-error">{error}</div>}
+    </div>
+  )
+}