@phenx-inc/ctlsurf 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/headless/index.mjs +339 -2
- package/out/headless/index.mjs.map +4 -4
- package/out/main/index.js +441 -20
- package/out/preload/index.js +9 -0
- package/out/renderer/assets/{cssMode-BQN8v2ok.js → cssMode-BbCVOTaJ.js} +3 -3
- package/out/renderer/assets/{freemarker2-DbxGYYVp.js → freemarker2-DryLywQv.js} +1 -1
- package/out/renderer/assets/{handlebars-3auU1CAd.js → handlebars-CdUMtSdr.js} +1 -1
- package/out/renderer/assets/{html-D8xFiRmI.js → html-Cwkf2GMH.js} +1 -1
- package/out/renderer/assets/{htmlMode-M3MApZ4n.js → htmlMode-C7PzhXxH.js} +3 -3
- package/out/renderer/assets/{index-B-iM7dFC.css → index-BIceGcz3.css} +52 -0
- package/out/renderer/assets/{index---H6cxNl.js → index-Dg1vMspf.js} +274 -244
- package/out/renderer/assets/{javascript-BO_ViZM5.js → javascript-Cy7ROXJl.js} +2 -2
- package/out/renderer/assets/{jsonMode-CKp2zvZu.js → jsonMode-BtZtt3QW.js} +3 -3
- package/out/renderer/assets/{liquid-C1eHcrht.js → liquid-xReAQeYs.js} +1 -1
- package/out/renderer/assets/{lspLanguageFeatures-CHWJx_Tl.js → lspLanguageFeatures-Dmx0L0Bw.js} +1 -1
- package/out/renderer/assets/{mdx-Qqdtk7fL.js → mdx-YMNFO1Zz.js} +1 -1
- package/out/renderer/assets/{python-DKu7rNbs.js → python-BExaDVxv.js} +1 -1
- package/out/renderer/assets/{razor-BOMpCo6z.js → razor-HL8ANmQ4.js} +1 -1
- package/out/renderer/assets/{transformers.web-DtSCnG36.js → transformers.web-_PuZoIho.js} +824 -19
- package/out/renderer/assets/{tsMode-yAjlPR-D.js → tsMode-Dqd8vdLE.js} +1 -1
- package/out/renderer/assets/{typescript-BiJRCUcL.js → typescript-Dysz1PTX.js} +1 -1
- package/out/renderer/assets/{xml-D4PvYeQq.js → xml-BHp5Z9nc.js} +1 -1
- package/out/renderer/assets/{yaml-BeHVkmnS.js → yaml-CPDHxVup.js} +1 -1
- package/out/renderer/index.html +64 -2
- package/out/repro-toggle.mjs +305 -0
- package/package.json +1 -1
- package/src/main/agentState.ts +329 -0
- package/src/main/index.ts +107 -19
- package/src/main/orchestrator.ts +34 -2
- package/src/main/transcripts.ts +134 -0
- package/src/preload/index.ts +11 -0
- package/src/renderer/App.tsx +48 -13
- package/src/renderer/components/FloatingMic.tsx +48 -0
- package/src/renderer/components/VoiceInput.tsx +36 -149
- package/src/renderer/index.html +62 -0
- package/src/renderer/lib/localWhisper.ts +46 -2
- package/src/renderer/lib/speech.ts +18 -0
- package/src/renderer/styles.css +52 -0
package/src/main/transcripts.ts
CHANGED
|
@@ -339,3 +339,137 @@ function capLength(str: string): string {
|
|
|
339
339
|
if (str.length <= MAX_ENTRY_CHARS) return str
|
|
340
340
|
return str.slice(0, MAX_ENTRY_CHARS) + `… [truncated, ${str.length} total chars]`
|
|
341
341
|
}
|
|
342
|
+
|
|
343
|
+
// ─── One-shot latest-reply read (on-demand "read aloud") ──────────
|
|
344
|
+
//
|
|
345
|
+
// Unlike the tailer, this does a single synchronous scan to return the most
|
|
346
|
+
// recent assistant reply for an agent + cwd, independent of whether the tailer
|
|
347
|
+
// is running. Used by the "read last response aloud" button so it works even
|
|
348
|
+
// when the speak-replies setting is off. Returns null when there's no
|
|
349
|
+
// transcript or no assistant message yet.
|
|
350
|
+
export function readLatestAssistantReply(opts: {
|
|
351
|
+
agentId: TranscriptAgentId
|
|
352
|
+
cwd: string
|
|
353
|
+
claudeProjectsDir?: string
|
|
354
|
+
codexSessionsDir?: string
|
|
355
|
+
}): string | null {
|
|
356
|
+
try {
|
|
357
|
+
const cwd = stripTrailingSep(opts.cwd)
|
|
358
|
+
if (opts.agentId === 'claude') {
|
|
359
|
+
const file = mostRecentClaudeFile(cwd, opts.claudeProjectsDir)
|
|
360
|
+
if (!file) return null
|
|
361
|
+
const text = lastClaudeAssistant(file, cwd)
|
|
362
|
+
return text ? capLength(text) : null
|
|
363
|
+
}
|
|
364
|
+
// Codex: the active session is the most recently written rollout for this cwd.
|
|
365
|
+
const codexRoot = opts.codexSessionsDir || path.join(os.homedir(), '.codex', 'sessions')
|
|
366
|
+
for (const file of recentCodexRollouts(codexRoot)) {
|
|
367
|
+
const text = lastCodexAgentMessage(file, cwd)
|
|
368
|
+
if (text) return capLength(text)
|
|
369
|
+
}
|
|
370
|
+
return null
|
|
371
|
+
} catch {
|
|
372
|
+
return null
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Whole-file read is fine here: transcripts are at most a few MB and we only do
|
|
377
|
+
// this on an explicit button click, not on a poll.
|
|
378
|
+
function readLines(file: string): string[] {
|
|
379
|
+
try {
|
|
380
|
+
return fs.readFileSync(file, 'utf-8').split('\n')
|
|
381
|
+
} catch {
|
|
382
|
+
return []
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function mostRecentClaudeFile(cwd: string, root?: string): string | null {
|
|
387
|
+
const projectsRoot = root || path.join(os.homedir(), '.claude', 'projects')
|
|
388
|
+
const slug = cwd.replace(/[^a-zA-Z0-9]/g, '-')
|
|
389
|
+
const dir = path.join(projectsRoot, slug)
|
|
390
|
+
let names: string[]
|
|
391
|
+
try {
|
|
392
|
+
names = fs.readdirSync(dir)
|
|
393
|
+
} catch {
|
|
394
|
+
return null
|
|
395
|
+
}
|
|
396
|
+
let best: string | null = null
|
|
397
|
+
let bestMs = -Infinity
|
|
398
|
+
for (const name of names) {
|
|
399
|
+
if (!name.endsWith('.jsonl')) continue
|
|
400
|
+
const fp = path.join(dir, name)
|
|
401
|
+
try {
|
|
402
|
+
const ms = fs.statSync(fp).mtimeMs
|
|
403
|
+
if (ms > bestMs) { bestMs = ms; best = fp }
|
|
404
|
+
} catch { /* ignore */ }
|
|
405
|
+
}
|
|
406
|
+
return best
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function recentCodexRollouts(root: string): string[] {
|
|
410
|
+
// Today and yesterday's session dirs (covers sessions spanning midnight),
|
|
411
|
+
// newest file first.
|
|
412
|
+
const dirs = new Set<string>()
|
|
413
|
+
const now = Date.now()
|
|
414
|
+
for (const ms of [now, now - 24 * 60 * 60 * 1000]) {
|
|
415
|
+
const d = new Date(ms)
|
|
416
|
+
const yyyy = String(d.getFullYear())
|
|
417
|
+
const mm = String(d.getMonth() + 1).padStart(2, '0')
|
|
418
|
+
const dd = String(d.getDate()).padStart(2, '0')
|
|
419
|
+
dirs.add(path.join(root, yyyy, mm, dd))
|
|
420
|
+
}
|
|
421
|
+
const files: Array<{ fp: string; ms: number }> = []
|
|
422
|
+
for (const dir of dirs) {
|
|
423
|
+
let names: string[]
|
|
424
|
+
try { names = fs.readdirSync(dir) } catch { continue }
|
|
425
|
+
for (const name of names) {
|
|
426
|
+
if (!name.startsWith('rollout-') || !name.endsWith('.jsonl')) continue
|
|
427
|
+
const fp = path.join(dir, name)
|
|
428
|
+
try { files.push({ fp, ms: fs.statSync(fp).mtimeMs }) } catch { /* ignore */ }
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return files.sort((a, b) => b.ms - a.ms).map((f) => f.fp)
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function lastClaudeAssistant(file: string, cwd: string): string | null {
|
|
435
|
+
const lines = readLines(file)
|
|
436
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
437
|
+
const trimmed = lines[i].trim()
|
|
438
|
+
if (!trimmed) continue
|
|
439
|
+
let obj: any
|
|
440
|
+
try { obj = JSON.parse(trimmed) } catch { continue }
|
|
441
|
+
if (!obj || typeof obj !== 'object') continue
|
|
442
|
+
if (obj.isMeta || obj.isSidechain || obj.type !== 'assistant') continue
|
|
443
|
+
if (typeof obj.cwd === 'string' && stripTrailingSep(obj.cwd) !== cwd) continue
|
|
444
|
+
const text = extractClaudeText(obj.message?.content, false)
|
|
445
|
+
if (text) return text
|
|
446
|
+
}
|
|
447
|
+
return null
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function lastCodexAgentMessage(file: string, cwd: string): string | null {
|
|
451
|
+
const lines = readLines(file)
|
|
452
|
+
// The date dir is shared across projects, so confirm this rollout belongs to
|
|
453
|
+
// our cwd via its session_meta before trusting any of its messages.
|
|
454
|
+
for (const line of lines) {
|
|
455
|
+
const trimmed = line.trim()
|
|
456
|
+
if (!trimmed) continue
|
|
457
|
+
let obj: any
|
|
458
|
+
try { obj = JSON.parse(trimmed) } catch { continue }
|
|
459
|
+
if (obj?.type === 'session_meta') {
|
|
460
|
+
const metaCwd = obj.payload?.cwd
|
|
461
|
+
if (typeof metaCwd === 'string' && stripTrailingSep(metaCwd) !== cwd) return null
|
|
462
|
+
break
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
466
|
+
const trimmed = lines[i].trim()
|
|
467
|
+
if (!trimmed) continue
|
|
468
|
+
let obj: any
|
|
469
|
+
try { obj = JSON.parse(trimmed) } catch { continue }
|
|
470
|
+
if (obj?.type !== 'event_msg' || obj.payload?.type !== 'agent_message') continue
|
|
471
|
+
const text = typeof obj.payload.message === 'string' ? obj.payload.message.trim() : ''
|
|
472
|
+
if (text && !isCodexNoise(text)) return text
|
|
473
|
+
}
|
|
474
|
+
return null
|
|
475
|
+
}
|
package/src/preload/index.ts
CHANGED
|
@@ -119,6 +119,17 @@ const api = {
|
|
|
119
119
|
ipcRenderer.on('agent:message', listener)
|
|
120
120
|
return () => ipcRenderer.removeListener('agent:message', listener)
|
|
121
121
|
},
|
|
122
|
+
// Real-time agent activity state for an in-app indicator dot.
|
|
123
|
+
// state: 'idle' | 'busy' | 'awaiting_input' | 'awaiting_decision'
|
|
124
|
+
onAgentState: (callback: (tabId: string, state: string, anyAwaiting: boolean) => void) => {
|
|
125
|
+
const listener = (_event: Electron.IpcRendererEvent, tabId: string, state: string, anyAwaiting: boolean) =>
|
|
126
|
+
callback(tabId, state, anyAwaiting)
|
|
127
|
+
ipcRenderer.on('agent:state', listener)
|
|
128
|
+
return () => ipcRenderer.removeListener('agent:state', listener)
|
|
129
|
+
},
|
|
130
|
+
// Latest assistant reply from the active tab's transcript (null if none).
|
|
131
|
+
getLastReply: (): Promise<string | null> =>
|
|
132
|
+
ipcRenderer.invoke('agent:getLastReply'),
|
|
122
133
|
|
|
123
134
|
// Filesystem
|
|
124
135
|
readDir: (dirPath: string): Promise<Array<{ name: string; path: string; isDirectory: boolean }>> =>
|
package/src/renderer/App.tsx
CHANGED
|
@@ -60,6 +60,8 @@ declare global {
|
|
|
60
60
|
getSpeakReplies: () => Promise<{ enabled: boolean }>
|
|
61
61
|
setSpeakReplies: (enabled: boolean) => Promise<{ enabled: boolean }>
|
|
62
62
|
onAgentMessage: (callback: (text: string) => void) => () => void
|
|
63
|
+
onAgentState: (callback: (tabId: string, state: string, anyAwaiting: boolean) => void) => () => void
|
|
64
|
+
getLastReply: () => Promise<string | null>
|
|
63
65
|
createProject: () => Promise<{ ok: boolean; folder_id?: string; error?: string }>
|
|
64
66
|
getWebviewInfo: () => Promise<{
|
|
65
67
|
frontendUrl: string; pageUrl?: string; authenticated: boolean;
|
|
@@ -136,6 +138,9 @@ export default function App() {
|
|
|
136
138
|
})
|
|
137
139
|
const [activeTabId, setActiveTabId] = useState<string>(tabs[0].id)
|
|
138
140
|
const [trackingActive, setTrackingActive] = useState(false)
|
|
141
|
+
// Per-tab agent activity ('busy' | 'awaiting_input' | 'awaiting_decision' | 'idle'),
|
|
142
|
+
// driven by the main process. Powers the awaiting-input dot on each tab.
|
|
143
|
+
const [agentStates, setAgentStates] = useState<Record<string, string>>({})
|
|
139
144
|
const [showTicketPanel, setShowTicketPanel] = useState(false)
|
|
140
145
|
// Draggable on-canvas push-to-talk mic; visibility persists across launches.
|
|
141
146
|
const [showFloatingMic, setShowFloatingMic] = useState<boolean>(() => {
|
|
@@ -155,16 +160,22 @@ export default function App() {
|
|
|
155
160
|
|
|
156
161
|
useEffect(() => {
|
|
157
162
|
async function init() {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
163
|
+
try {
|
|
164
|
+
const list = await window.worker.listAgents()
|
|
165
|
+
setAgents(list)
|
|
166
|
+
const status = await window.worker.getWorkerStatus()
|
|
167
|
+
setWsStatus(status)
|
|
168
|
+
// Load initial cwd for the picker
|
|
169
|
+
const initialCwd = await window.worker.getCwd().catch(() => window.worker.getHomePath())
|
|
170
|
+
setCwd(initialCwd)
|
|
171
|
+
cwdRef.current = initialCwd
|
|
172
|
+
// Connected ctlsurf project name for the header (null until a worker registers)
|
|
173
|
+
window.worker.getProjectName().then(setProjectName).catch(() => { /* ignore */ })
|
|
174
|
+
} finally {
|
|
175
|
+
// Dismiss the boot splash painted by index.html now that startup IPC has
|
|
176
|
+
// settled. In `finally` so a failed IPC call never leaves it stuck on.
|
|
177
|
+
;(window as unknown as { __dismissSplash?: () => void }).__dismissSplash?.()
|
|
178
|
+
}
|
|
168
179
|
}
|
|
169
180
|
init()
|
|
170
181
|
}, [])
|
|
@@ -195,6 +206,16 @@ export default function App() {
|
|
|
195
206
|
return unsub
|
|
196
207
|
}, [])
|
|
197
208
|
|
|
209
|
+
// Track each tab's agent activity for the awaiting-input dot. The aggregate-
|
|
210
|
+
// only signal arrives with an empty tabId, which we ignore here.
|
|
211
|
+
useEffect(() => {
|
|
212
|
+
const unsub = window.worker.onAgentState((tabId, state) => {
|
|
213
|
+
if (!tabId) return
|
|
214
|
+
setAgentStates(prev => (prev[tabId] === state ? prev : { ...prev, [tabId]: state }))
|
|
215
|
+
})
|
|
216
|
+
return unsub
|
|
217
|
+
}, [])
|
|
218
|
+
|
|
198
219
|
// Reflect tracking state for the active tab; refresh on tab switch and poll
|
|
199
220
|
// periodically so the UI catches changes (e.g. a session ending naturally).
|
|
200
221
|
useEffect(() => {
|
|
@@ -225,8 +246,15 @@ export default function App() {
|
|
|
225
246
|
const handleVoiceTranscript = useCallback((text: string) => {
|
|
226
247
|
const trimmed = text.trim()
|
|
227
248
|
if (!trimmed) return
|
|
228
|
-
|
|
249
|
+
// Send the text and the submit Enter as TWO separate writes. A long transcript
|
|
250
|
+
// glued to its trailing \r arrives as one fast burst, which the agent's input
|
|
251
|
+
// box treats as a paste and turns the carriage return into a literal newline
|
|
252
|
+
// instead of submitting. Delaying the Enter lets the text be consumed first so
|
|
253
|
+
// \r registers as a real submit keystroke. (80ms is the tunable knob if very
|
|
254
|
+
// long input ever needs more time to drain.)
|
|
255
|
+
window.worker.writePty(activeTabId, trimmed)
|
|
229
256
|
focusTerminal(activeTabId)
|
|
257
|
+
setTimeout(() => window.worker.writePty(activeTabId, '\r'), 80)
|
|
230
258
|
}, [activeTabId])
|
|
231
259
|
|
|
232
260
|
const cwdRef = useRef<string | null>(null)
|
|
@@ -410,8 +438,15 @@ export default function App() {
|
|
|
410
438
|
>
|
|
411
439
|
<span className="terminal-tab-label">
|
|
412
440
|
{tab.agent ? tab.agent.name : tab.label}
|
|
413
|
-
{tab.agentStatus === '
|
|
414
|
-
|
|
441
|
+
{tab.agentStatus === 'exited' ? (
|
|
442
|
+
<span className="terminal-tab-dot exited" />
|
|
443
|
+
) : agentStates[tab.id] === 'awaiting_decision' ? (
|
|
444
|
+
<span className="terminal-tab-dot decision" title="Awaiting your approval" />
|
|
445
|
+
) : agentStates[tab.id] === 'awaiting_input' ? (
|
|
446
|
+
<span className="terminal-tab-dot awaiting" title="Awaiting your input" />
|
|
447
|
+
) : tab.agentStatus === 'active' ? (
|
|
448
|
+
<span className="terminal-tab-dot active" />
|
|
449
|
+
) : null}
|
|
415
450
|
</span>
|
|
416
451
|
{tabs.length > 1 && (
|
|
417
452
|
<button
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef, useState } from 'react'
|
|
2
2
|
import { VoiceInput } from './VoiceInput'
|
|
3
|
+
import { speech } from '../lib/speech'
|
|
3
4
|
|
|
4
5
|
// A draggable, dismissable push-to-talk mic that floats over the panes. It wraps
|
|
5
6
|
// the same <VoiceInput> push-to-talk logic used in the titlebar; only the chrome
|
|
@@ -36,6 +37,29 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
|
|
|
36
37
|
// Pointer-to-element offset captured at drag start; null when not dragging.
|
|
37
38
|
const dragRef = useRef<{ dx: number; dy: number } | null>(null)
|
|
38
39
|
|
|
40
|
+
// Read-aloud (TTS) of the last agent reply. `speaking` mirrors the shared
|
|
41
|
+
// speech singleton; `note` is a transient message (e.g. nothing to read yet).
|
|
42
|
+
const [speaking, setSpeaking] = useState(false)
|
|
43
|
+
const [note, setNote] = useState<string | null>(null)
|
|
44
|
+
useEffect(() => speech.onActivity(setSpeaking), [])
|
|
45
|
+
useEffect(() => {
|
|
46
|
+
if (!note) return
|
|
47
|
+
const t = setTimeout(() => setNote(null), 3000)
|
|
48
|
+
return () => clearTimeout(t)
|
|
49
|
+
}, [note])
|
|
50
|
+
|
|
51
|
+
const onReadAloud = useCallback(async () => {
|
|
52
|
+
// Toggle: a second click while speaking stops it.
|
|
53
|
+
if (speaking) { speech.stop(); return }
|
|
54
|
+
// Prime the audio context from this click (synchronously, before the await)
|
|
55
|
+
// so neural playback isn't blocked by the autoplay policy.
|
|
56
|
+
speech.unlock()
|
|
57
|
+
let text: string | null = null
|
|
58
|
+
try { text = await window.worker.getLastReply() } catch { /* ignore */ }
|
|
59
|
+
if (text) speech.enqueue(text)
|
|
60
|
+
else setNote('No reply to read yet')
|
|
61
|
+
}, [speaking])
|
|
62
|
+
|
|
39
63
|
// Keep the button fully inside the viewport (used on drag, mount, and resize).
|
|
40
64
|
const clamp = useCallback((x: number, y: number): Pos => {
|
|
41
65
|
const el = elRef.current
|
|
@@ -123,6 +147,30 @@ export function FloatingMic({ onTranscript, onHide }: FloatingMicProps) {
|
|
|
123
147
|
</button>
|
|
124
148
|
</div>
|
|
125
149
|
<VoiceInput variant="floating" onTranscript={onTranscript} />
|
|
150
|
+
<button
|
|
151
|
+
type="button"
|
|
152
|
+
className={`floating-mic-read ${speaking ? 'speaking' : ''}`}
|
|
153
|
+
// Don't let a click on the button start a drag.
|
|
154
|
+
onPointerDown={(e) => e.stopPropagation()}
|
|
155
|
+
onClick={onReadAloud}
|
|
156
|
+
title={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
|
|
157
|
+
aria-label={speaking ? 'Stop reading' : 'Read last agent reply aloud'}
|
|
158
|
+
aria-pressed={speaking}
|
|
159
|
+
>
|
|
160
|
+
{speaking ? (
|
|
161
|
+
<svg viewBox="0 0 24 24" width="14" height="14" fill="currentColor" aria-hidden="true">
|
|
162
|
+
<rect x="6" y="6" width="12" height="12" rx="2" />
|
|
163
|
+
</svg>
|
|
164
|
+
) : (
|
|
165
|
+
<svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor"
|
|
166
|
+
strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
|
|
167
|
+
<polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5" />
|
|
168
|
+
<path d="M15.54 8.46a5 5 0 0 1 0 7.07" />
|
|
169
|
+
<path d="M19.07 4.93a10 10 0 0 1 0 14.14" />
|
|
170
|
+
</svg>
|
|
171
|
+
)}
|
|
172
|
+
</button>
|
|
173
|
+
{note && <div className="floating-mic-note">{note}</div>}
|
|
126
174
|
</div>
|
|
127
175
|
)
|
|
128
176
|
}
|
|
@@ -1,76 +1,26 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef, useState } from 'react'
|
|
2
2
|
import { transcribeBlob, type ModelProgress } from '../lib/localWhisper'
|
|
3
3
|
|
|
4
|
-
// ───
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
|
|
10
|
-
interface SpeechRecognitionResult {
|
|
11
|
-
isFinal: boolean
|
|
12
|
-
0: { transcript: string }
|
|
13
|
-
}
|
|
14
|
-
interface SpeechRecognitionEvent {
|
|
15
|
-
resultIndex: number
|
|
16
|
-
results: { length: number;[index: number]: SpeechRecognitionResult }
|
|
17
|
-
}
|
|
18
|
-
interface SpeechRecognitionErrorEvent { error: string }
|
|
19
|
-
interface SpeechRecognitionLike {
|
|
20
|
-
lang: string
|
|
21
|
-
continuous: boolean
|
|
22
|
-
interimResults: boolean
|
|
23
|
-
start: () => void
|
|
24
|
-
stop: () => void
|
|
25
|
-
abort: () => void
|
|
26
|
-
onresult: ((e: SpeechRecognitionEvent) => void) | null
|
|
27
|
-
onerror: ((e: SpeechRecognitionErrorEvent) => void) | null
|
|
28
|
-
onend: (() => void) | null
|
|
29
|
-
}
|
|
30
|
-
type SpeechRecognitionCtor = new () => SpeechRecognitionLike
|
|
4
|
+
// ─── Capabilities ────────────────────────────────────
|
|
5
|
+
// Speech is transcribed entirely on-device via a local Whisper model (see
|
|
6
|
+
// ../lib/localWhisper). The browser's Web Speech API is intentionally not used:
|
|
7
|
+
// it's a cloud service whose Google backend is unreachable inside Electron
|
|
8
|
+
// (Chromium ships without the API key), so it never works in this app.
|
|
31
9
|
|
|
32
|
-
function getRecognitionCtor(): SpeechRecognitionCtor | null {
|
|
33
|
-
const w = window as unknown as {
|
|
34
|
-
SpeechRecognition?: SpeechRecognitionCtor
|
|
35
|
-
webkitSpeechRecognition?: SpeechRecognitionCtor
|
|
36
|
-
}
|
|
37
|
-
return w.SpeechRecognition || w.webkitSpeechRecognition || null
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// ─── Capabilities & engine selection ─────────────────
|
|
41
|
-
|
|
42
|
-
type Engine = 'web-speech' | 'local'
|
|
43
10
|
type Phase = 'idle' | 'listening' | 'transcribing'
|
|
44
11
|
|
|
45
|
-
|
|
46
|
-
// Persisted audioinput deviceId for the local engine; null/absent = OS default.
|
|
12
|
+
// Persisted audioinput deviceId; null/absent = OS default.
|
|
47
13
|
const DEVICE_KEY = 'ctlsurf.voiceDeviceId'
|
|
48
14
|
|
|
49
15
|
function loadDeviceId(): string | null {
|
|
50
16
|
try { return localStorage.getItem(DEVICE_KEY) || null } catch { return null }
|
|
51
17
|
}
|
|
52
18
|
|
|
53
|
-
const WEB_SPEECH_SUPPORTED = getRecognitionCtor() !== null
|
|
54
19
|
const LOCAL_SUPPORTED =
|
|
55
20
|
typeof navigator !== 'undefined' &&
|
|
56
21
|
!!navigator.mediaDevices?.getUserMedia &&
|
|
57
22
|
typeof MediaRecorder !== 'undefined' &&
|
|
58
23
|
typeof OfflineAudioContext !== 'undefined'
|
|
59
|
-
const ANY_SUPPORTED = WEB_SPEECH_SUPPORTED || LOCAL_SUPPORTED
|
|
60
|
-
|
|
61
|
-
function loadInitialEngine(): Engine {
|
|
62
|
-
if (!WEB_SPEECH_SUPPORTED && LOCAL_SUPPORTED) return 'local'
|
|
63
|
-
try {
|
|
64
|
-
if (localStorage.getItem(ENGINE_KEY) === 'local' && LOCAL_SUPPORTED) return 'local'
|
|
65
|
-
} catch { /* ignore */ }
|
|
66
|
-
return WEB_SPEECH_SUPPORTED ? 'web-speech' : 'local'
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// Web Speech errors that mean the engine itself is unreachable (vs. a mic
|
|
70
|
-
// permission/hardware problem, which would also break the local fallback).
|
|
71
|
-
function isEngineUnavailable(code: string): boolean {
|
|
72
|
-
return code === 'network' || code === 'service-not-allowed'
|
|
73
|
-
}
|
|
74
24
|
|
|
75
25
|
function describeMicError(err: unknown): string {
|
|
76
26
|
const name = (err as { name?: string })?.name
|
|
@@ -88,14 +38,14 @@ interface VoiceInputProps {
|
|
|
88
38
|
}
|
|
89
39
|
|
|
90
40
|
export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputProps) {
|
|
91
|
-
const [engine, setEngine] = useState<Engine>(loadInitialEngine)
|
|
92
41
|
const [phase, setPhase] = useState<Phase>('idle')
|
|
93
|
-
const [interim, setInterim] = useState('')
|
|
94
42
|
const [modelPct, setModelPct] = useState<number | null>(null)
|
|
43
|
+
// Transcript-so-far streamed from the model during transcription (live preview).
|
|
44
|
+
const [partial, setPartial] = useState('')
|
|
95
45
|
const [error, setError] = useState<string | null>(null)
|
|
96
46
|
const [notice, setNotice] = useState<string | null>(null)
|
|
97
47
|
|
|
98
|
-
// Mic source selection
|
|
48
|
+
// Mic source selection
|
|
99
49
|
const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
|
|
100
50
|
const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(loadDeviceId)
|
|
101
51
|
const [showDevicePicker, setShowDevicePicker] = useState(false)
|
|
@@ -103,19 +53,13 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
103
53
|
useEffect(() => { selectedDeviceIdRef.current = selectedDeviceId }, [selectedDeviceId])
|
|
104
54
|
const wrapRef = useRef<HTMLDivElement>(null)
|
|
105
55
|
|
|
106
|
-
//
|
|
107
|
-
const recognitionRef = useRef<SpeechRecognitionLike | null>(null)
|
|
108
|
-
const finalRef = useRef('')
|
|
109
|
-
// Local (Whisper) refs
|
|
56
|
+
// Recording refs
|
|
110
57
|
const streamRef = useRef<MediaStream | null>(null)
|
|
111
58
|
const recorderRef = useRef<MediaRecorder | null>(null)
|
|
112
59
|
const chunksRef = useRef<Blob[]>([])
|
|
113
60
|
// Set true when the user releases before getUserMedia resolves (quick tap).
|
|
114
61
|
const cancelGestureRef = useRef(false)
|
|
115
62
|
|
|
116
|
-
const engineRef = useRef(engine)
|
|
117
|
-
useEffect(() => { engineRef.current = engine }, [engine])
|
|
118
|
-
|
|
119
63
|
const onTranscriptRef = useRef(onTranscript)
|
|
120
64
|
useEffect(() => { onTranscriptRef.current = onTranscript }, [onTranscript])
|
|
121
65
|
|
|
@@ -131,12 +75,6 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
131
75
|
return () => clearTimeout(t)
|
|
132
76
|
}, [notice])
|
|
133
77
|
|
|
134
|
-
const switchToLocal = useCallback((reason: string) => {
|
|
135
|
-
try { localStorage.setItem(ENGINE_KEY, 'local') } catch { /* ignore */ }
|
|
136
|
-
setEngine('local')
|
|
137
|
-
setNotice(reason)
|
|
138
|
-
}, [])
|
|
139
|
-
|
|
140
78
|
const stopStream = useCallback(() => {
|
|
141
79
|
streamRef.current?.getTracks().forEach((t) => t.stop())
|
|
142
80
|
streamRef.current = null
|
|
@@ -224,66 +162,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
224
162
|
return () => document.removeEventListener('mousedown', onDocDown)
|
|
225
163
|
}, [showDevicePicker])
|
|
226
164
|
|
|
227
|
-
// ───
|
|
228
|
-
|
|
229
|
-
const startWebSpeech = useCallback(() => {
|
|
230
|
-
const Ctor = getRecognitionCtor()
|
|
231
|
-
if (!Ctor || recognitionRef.current) return
|
|
232
|
-
setError(null); setNotice(null); setInterim('')
|
|
233
|
-
finalRef.current = ''
|
|
234
|
-
|
|
235
|
-
const rec = new Ctor()
|
|
236
|
-
rec.lang = navigator.language || 'en-US'
|
|
237
|
-
rec.continuous = true
|
|
238
|
-
rec.interimResults = true
|
|
239
|
-
|
|
240
|
-
rec.onresult = (event) => {
|
|
241
|
-
let finalText = ''
|
|
242
|
-
let interimText = ''
|
|
243
|
-
for (let i = 0; i < event.results.length; i++) {
|
|
244
|
-
const res = event.results[i]
|
|
245
|
-
if (res.isFinal) finalText += res[0].transcript
|
|
246
|
-
else interimText += res[0].transcript
|
|
247
|
-
}
|
|
248
|
-
finalRef.current = finalText
|
|
249
|
-
setInterim(interimText)
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
rec.onerror = (event) => {
|
|
253
|
-
if (isEngineUnavailable(event.error) && LOCAL_SUPPORTED) {
|
|
254
|
-
// The streamed audio is gone; switch engines and ask for a retry.
|
|
255
|
-
finalRef.current = ''
|
|
256
|
-
switchToLocal('Voice service unavailable — switched to on-device. Press again.')
|
|
257
|
-
} else if (event.error !== 'no-speech' && event.error !== 'aborted') {
|
|
258
|
-
setError(event.error === 'not-allowed' ? 'Microphone access denied' : `Voice error: ${event.error}`)
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
rec.onend = () => {
|
|
263
|
-
const text = finalRef.current.trim()
|
|
264
|
-
recognitionRef.current = null
|
|
265
|
-
setPhase('idle')
|
|
266
|
-
setInterim('')
|
|
267
|
-
if (text) onTranscriptRef.current(text)
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
recognitionRef.current = rec
|
|
271
|
-
try {
|
|
272
|
-
rec.start()
|
|
273
|
-
setPhase('listening')
|
|
274
|
-
} catch (err) {
|
|
275
|
-
recognitionRef.current = null
|
|
276
|
-
setPhase('idle')
|
|
277
|
-
setError('Could not start microphone')
|
|
278
|
-
console.error('[voice] web speech start failed', err)
|
|
279
|
-
}
|
|
280
|
-
}, [switchToLocal])
|
|
281
|
-
|
|
282
|
-
const stopWebSpeech = useCallback(() => {
|
|
283
|
-
try { recognitionRef.current?.stop() } catch { /* already stopped */ }
|
|
284
|
-
}, [])
|
|
285
|
-
|
|
286
|
-
// ─── Local (Whisper) engine ────────────────────────
|
|
165
|
+
// ─── On-device (Whisper) transcription ─────────────
|
|
287
166
|
|
|
288
167
|
const handleModelProgress = useCallback((p: ModelProgress) => {
|
|
289
168
|
if (p.status === 'progress' && typeof p.progress === 'number') {
|
|
@@ -299,9 +178,14 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
299
178
|
if (blob.size === 0) { setPhase('idle'); return }
|
|
300
179
|
|
|
301
180
|
setPhase('transcribing')
|
|
302
|
-
|
|
181
|
+
setPartial('')
|
|
303
182
|
try {
|
|
304
|
-
const text = await transcribeBlob(blob, handleModelProgress)
|
|
183
|
+
const text = await transcribeBlob(blob, handleModelProgress, (t) => {
|
|
184
|
+
// First streamed token means the model is loaded and decoding — drop the
|
|
185
|
+
// download indicator and start showing the live transcript.
|
|
186
|
+
setModelPct(null)
|
|
187
|
+
setPartial(t)
|
|
188
|
+
})
|
|
305
189
|
if (text) onTranscriptRef.current(text)
|
|
306
190
|
else setNotice('No speech detected — check the mic source (▾).')
|
|
307
191
|
} catch (err) {
|
|
@@ -310,11 +194,12 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
310
194
|
} finally {
|
|
311
195
|
setPhase('idle')
|
|
312
196
|
setModelPct(null)
|
|
197
|
+
setPartial('')
|
|
313
198
|
}
|
|
314
199
|
}, [stopStream, handleModelProgress])
|
|
315
200
|
|
|
316
201
|
const startLocal = useCallback(async () => {
|
|
317
|
-
setError(null); setNotice(null)
|
|
202
|
+
setError(null); setNotice(null)
|
|
318
203
|
cancelGestureRef.current = false
|
|
319
204
|
try {
|
|
320
205
|
const stream = await getStream()
|
|
@@ -351,21 +236,18 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
351
236
|
// ─── Push-to-talk gesture ──────────────────────────
|
|
352
237
|
|
|
353
238
|
const handlePointerDown = (e: React.PointerEvent) => {
|
|
354
|
-
if (!
|
|
239
|
+
if (!LOCAL_SUPPORTED || phase !== 'idle') return
|
|
355
240
|
e.preventDefault()
|
|
356
241
|
e.currentTarget.setPointerCapture?.(e.pointerId)
|
|
357
|
-
|
|
358
|
-
else if (LOCAL_SUPPORTED) void startLocal()
|
|
242
|
+
void startLocal()
|
|
359
243
|
}
|
|
360
244
|
const handlePointerUp = (e: React.PointerEvent) => {
|
|
361
245
|
e.currentTarget.releasePointerCapture?.(e.pointerId)
|
|
362
|
-
|
|
363
|
-
else stopLocal()
|
|
246
|
+
stopLocal()
|
|
364
247
|
}
|
|
365
248
|
|
|
366
249
|
// Clean up on unmount.
|
|
367
250
|
useEffect(() => () => {
|
|
368
|
-
try { recognitionRef.current?.abort() } catch { /* ignore */ }
|
|
369
251
|
try { recorderRef.current?.stop() } catch { /* ignore */ }
|
|
370
252
|
streamRef.current?.getTracks().forEach((t) => t.stop())
|
|
371
253
|
}, [])
|
|
@@ -375,21 +257,27 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
375
257
|
const listening = phase === 'listening'
|
|
376
258
|
const busy = phase === 'transcribing'
|
|
377
259
|
|
|
378
|
-
const title = !
|
|
260
|
+
const title = !LOCAL_SUPPORTED
|
|
379
261
|
? 'Voice typing not supported in this build'
|
|
380
262
|
: listening
|
|
381
263
|
? 'Listening… release to insert'
|
|
382
264
|
: busy
|
|
383
265
|
? 'Transcribing…'
|
|
384
|
-
:
|
|
385
|
-
? 'Hold to talk (on-device) — speech is typed into the terminal'
|
|
386
|
-
: 'Hold to talk — speech is typed into the terminal'
|
|
266
|
+
: 'Hold to talk (on-device) — speech is typed into the terminal'
|
|
387
267
|
|
|
388
268
|
let chip: { kind: 'listening' | 'busy' | 'notice' | 'error'; text: string } | null = null
|
|
389
269
|
if (error && phase === 'idle') chip = { kind: 'error', text: error }
|
|
390
270
|
else if (notice && phase === 'idle') chip = { kind: 'notice', text: notice }
|
|
391
|
-
else if (listening) chip = { kind: 'listening', text:
|
|
392
|
-
else if (busy)
|
|
271
|
+
else if (listening) chip = { kind: 'listening', text: 'Recording…' }
|
|
272
|
+
else if (busy) {
|
|
273
|
+
// Show the most recent ~90 chars of the live transcript as a ticker so a long
|
|
274
|
+
// dictation doesn't overflow the chip.
|
|
275
|
+
const preview = partial.length > 90 ? '…' + partial.slice(-90) : partial
|
|
276
|
+
chip = {
|
|
277
|
+
kind: 'busy',
|
|
278
|
+
text: modelPct !== null ? `Downloading voice model… ${modelPct}%` : (preview || 'Transcribing…'),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
393
281
|
|
|
394
282
|
const floating = variant === 'floating'
|
|
395
283
|
const btnClass = floating
|
|
@@ -408,7 +296,6 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
408
296
|
const activeLabel = selectedDeviceId
|
|
409
297
|
? (realDevices.find((d) => d.deviceId === selectedDeviceId)?.label || 'Selected microphone')
|
|
410
298
|
: defaultLabel
|
|
411
|
-
// Source selection only affects the local (getUserMedia) engine.
|
|
412
299
|
const showSourcePicker = floating && LOCAL_SUPPORTED
|
|
413
300
|
|
|
414
301
|
return (
|
|
@@ -416,7 +303,7 @@ export function VoiceInput({ onTranscript, variant = 'titlebar' }: VoiceInputPro
|
|
|
416
303
|
<button
|
|
417
304
|
type="button"
|
|
418
305
|
className={btnClass}
|
|
419
|
-
disabled={!
|
|
306
|
+
disabled={!LOCAL_SUPPORTED}
|
|
420
307
|
onPointerDown={handlePointerDown}
|
|
421
308
|
onPointerUp={handlePointerUp}
|
|
422
309
|
onPointerCancel={handlePointerUp}
|