@vaibhavgupta0786/opencode-voice 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +12 -0
- package/README.md +138 -2
- package/bin/SHA256SUMS +3 -0
- package/bin/darwin-arm64/opencode-voice-recorder +0 -0
- package/bin/darwin-x64/opencode-voice-recorder +0 -0
- package/bin/linux-x64/opencode-voice-recorder +0 -0
- package/docs/STT.md +64 -0
- package/docs/TROUBLESHOOTING.md +82 -0
- package/docs/VOICE-MODE.md +42 -0
- package/logic.ts +171 -0
- package/package.json +36 -4
- package/servers/mlx_server.py +94 -0
- package/skills/voice-check/SKILL.md +68 -0
- package/tui.tsx +732 -0
package/tui.tsx
ADDED
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
import { Plugin } from "@opencode/plugin/tui"
|
|
2
|
+
import { createSignal } from "solid-js"
|
|
3
|
+
import { spawn, type ChildProcess } from "node:child_process"
|
|
4
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
5
|
+
import { homedir } from "node:os"
|
|
6
|
+
import { join } from "node:path"
|
|
7
|
+
import { fileURLToPath } from "node:url"
|
|
8
|
+
import {
|
|
9
|
+
RATE,
|
|
10
|
+
TICK_MS,
|
|
11
|
+
hadSpeech,
|
|
12
|
+
healthEndpoints,
|
|
13
|
+
initialVadState,
|
|
14
|
+
isLocalEndpoint,
|
|
15
|
+
mergeOptions,
|
|
16
|
+
peakOf,
|
|
17
|
+
sttEndpoint,
|
|
18
|
+
vadStep,
|
|
19
|
+
wavFromPcm,
|
|
20
|
+
type Capture,
|
|
21
|
+
type VoiceOptions,
|
|
22
|
+
} from "./logic.ts"
|
|
23
|
+
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
// opencode-voice: local-only push-to-talk for OpenCode v2.
|
|
26
|
+
// F9 — dictate a take (press to start, press to stop); takes append to
|
|
27
|
+
// that session's draft
|
|
28
|
+
// F10 — open the session's draft editor (any time, empty or not);
|
|
29
|
+
// Ctrl+Enter sends, Esc keeps edits
|
|
30
|
+
// F12 — wipe the session's draft (confirmation dialog)
|
|
31
|
+
// Each session keeps its own draft, so switching chats never mixes them.
|
|
32
|
+
// On MacBooks press F-keys with Fn: plain F10 is the hardware mic-mute key,
|
|
33
|
+
// plain F11 is Show Desktop (macOS reserves it), so review/wipe use F10/F12.
|
|
34
|
+
// No cloud, no temp audio files, no auto-send, no permission answering.
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
type Phase = "idle" | "recording" | "transcribing"
|
|
38
|
+
|
|
39
|
+
/** STT server health. Only tracked for local endpoints — cloud STT has no
|
|
40
|
+
* health route, so a dot there would always lie (see isLocalEndpoint). */
|
|
41
|
+
type ServerHealth = "unknown" | "up" | "down"
|
|
42
|
+
|
|
43
|
+
const HEALTH_POLL_MS = 30_000
|
|
44
|
+
const HEALTH_TIMEOUT_MS = 2_000
|
|
45
|
+
const HEALTH_DEBOUNCE_MS = 5_000
|
|
46
|
+
|
|
47
|
+
const DEBUG_LOG = "/tmp/opencode/voice-plugin.log"
|
|
48
|
+
|
|
49
|
+
function dbg(enabled: boolean, message: string): void {
|
|
50
|
+
if (!enabled) return
|
|
51
|
+
try {
|
|
52
|
+
mkdirSync("/tmp/opencode", { recursive: true })
|
|
53
|
+
appendFileSync(DEBUG_LOG, `${new Date().toISOString()} ${message}\n`)
|
|
54
|
+
} catch {
|
|
55
|
+
// logging must never break dictation
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function here(): string {
|
|
60
|
+
try {
|
|
61
|
+
return join(fileURLToPath(new URL(".", import.meta.url)))
|
|
62
|
+
} catch {
|
|
63
|
+
return join(process.env.HOME ?? "~", ".config", "opencode", "plugins", "voice")
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Bundled mic recorder (audited source: miniaudio, streams raw PCM to stdout). */
|
|
68
|
+
function recorderPath(): string | null {
|
|
69
|
+
const os = process.platform === "darwin" ? "darwin" : process.platform === "linux" ? "linux" : null
|
|
70
|
+
if (!os) return null
|
|
71
|
+
const arch = process.arch === "arm64" ? "arm64" : "x64"
|
|
72
|
+
const path = join(here(), "bin", `${os}-${arch}`, "opencode-voice-recorder")
|
|
73
|
+
return existsSync(path) ? path : null
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Path of the voice-check skill if installed in the user's global skills
|
|
77
|
+
* dir (skills are only discovered from there, not from plugin packages). */
|
|
78
|
+
function installedSkillPath(): string {
|
|
79
|
+
const configHome = process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config")
|
|
80
|
+
return join(configHome, "opencode", "skills", "voice-check", "SKILL.md")
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Install the bundled voice-check skill into the user's global skills dir
|
|
84
|
+
* (once — an existing file is never overwritten, so user edits survive).
|
|
85
|
+
* Only ever called from the explicit voice-setup command: a plugin package
|
|
86
|
+
* silently writing to global config on load would be exactly the kind of
|
|
87
|
+
* behavior a careful developer audits for. */
|
|
88
|
+
function installSkill(log: (m: string) => void): "installed" | "exists" | "missing-source" | "error" {
|
|
89
|
+
try {
|
|
90
|
+
const target = installedSkillPath()
|
|
91
|
+
if (existsSync(target)) return "exists"
|
|
92
|
+
const source = join(here(), "skills", "voice-check", "SKILL.md")
|
|
93
|
+
if (!existsSync(source)) return "missing-source"
|
|
94
|
+
mkdirSync(join(target, ".."), { recursive: true })
|
|
95
|
+
writeFileSync(target, readFileSync(source))
|
|
96
|
+
log("installed voice-check skill globally")
|
|
97
|
+
return "installed"
|
|
98
|
+
} catch {
|
|
99
|
+
// a skill install must never break dictation
|
|
100
|
+
return "error"
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Record one utterance. Audio stays in memory.
|
|
105
|
+
* Returns the capture promise plus a stop handle for toggle mode.
|
|
106
|
+
* `shouldAbort` is polled every 500 ms: the moment it reports true the
|
|
107
|
+
* recorder is killed and the take is discarded ("take-aborted"). */
|
|
108
|
+
function capture(
|
|
109
|
+
opts: ReturnType<typeof mergeOptions>,
|
|
110
|
+
log: (m: string) => void,
|
|
111
|
+
shouldAbort?: () => boolean,
|
|
112
|
+
): { done: Promise<Capture>; stop: () => void } {
|
|
113
|
+
const cmd = recorderPath()
|
|
114
|
+
if (!cmd)
|
|
115
|
+
return {
|
|
116
|
+
done: Promise.reject(new Error("no bundled recorder for this platform")),
|
|
117
|
+
stop: () => {},
|
|
118
|
+
}
|
|
119
|
+
const maxSec = Math.ceil(opts.maxMs / 1000).toFixed(0)
|
|
120
|
+
|
|
121
|
+
let stopFn: () => void = () => {}
|
|
122
|
+
const done = new Promise<Capture>((resolve, reject) => {
|
|
123
|
+
const child: ChildProcess = spawn(cmd, ["--seconds", maxSec, "--gain", "1.00"], {
|
|
124
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
125
|
+
})
|
|
126
|
+
const chunks: Uint8Array[] = []
|
|
127
|
+
let state = initialVadState()
|
|
128
|
+
let finished = false
|
|
129
|
+
let errorText = ""
|
|
130
|
+
let leftover: Uint8Array = new Uint8Array(0)
|
|
131
|
+
|
|
132
|
+
const finish = (reason: string) => {
|
|
133
|
+
if (finished) return
|
|
134
|
+
finished = true
|
|
135
|
+
stopFn = () => {}
|
|
136
|
+
clearInterval(timer)
|
|
137
|
+
try {
|
|
138
|
+
child.kill("SIGKILL")
|
|
139
|
+
} catch {
|
|
140
|
+
// already exited
|
|
141
|
+
}
|
|
142
|
+
const raw = Buffer.concat(chunks)
|
|
143
|
+
if (leftover.length > 0) {
|
|
144
|
+
// account for a trailing partial window
|
|
145
|
+
const peak = peakOf(leftover)
|
|
146
|
+
if (peak > opts.vadThreshold) state = { ...state, voicedMs: state.voicedMs + (leftover.length / (RATE * 2)) * 1000 }
|
|
147
|
+
}
|
|
148
|
+
if (raw.length === 0 && errorText.trim()) {
|
|
149
|
+
reject(new Error(`recorder produced no audio: ${errorText.trim().slice(0, 200)}`))
|
|
150
|
+
return
|
|
151
|
+
}
|
|
152
|
+
log(
|
|
153
|
+
`capture ${reason} voiced=${Math.round(state.voicedMs)}ms loudest=${state.loudest.toFixed(3)} bytes=${raw.length}`,
|
|
154
|
+
)
|
|
155
|
+
resolve({ wav: wavFromPcm(raw), voicedMs: state.voicedMs, loudest: state.loudest, hadSpeech: hadSpeech(state, opts) })
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
child.stdout?.on("data", (data: Buffer) => {
|
|
159
|
+
if (finished) return
|
|
160
|
+
chunks.push(data)
|
|
161
|
+
let buf: Uint8Array = leftover.length > 0 ? Buffer.concat([leftover, data]) : data
|
|
162
|
+
const window = Math.floor((RATE * 2 * TICK_MS) / 1000)
|
|
163
|
+
while (buf.length >= window && !finished) {
|
|
164
|
+
state = vadStep(state, peakOf(buf.subarray(0, window)), opts)
|
|
165
|
+
buf = buf.subarray(window)
|
|
166
|
+
if (state.done) finish(state.doneReason ?? "vad")
|
|
167
|
+
}
|
|
168
|
+
leftover = buf
|
|
169
|
+
})
|
|
170
|
+
child.stderr?.on("data", (data: Buffer) => {
|
|
171
|
+
if (errorText.length < 2048) errorText += data.toString()
|
|
172
|
+
})
|
|
173
|
+
child.on("error", (error) => {
|
|
174
|
+
if (!finished) {
|
|
175
|
+
finished = true
|
|
176
|
+
clearInterval(timer)
|
|
177
|
+
reject(error)
|
|
178
|
+
}
|
|
179
|
+
})
|
|
180
|
+
child.on("close", () => finish("recorder-exit"))
|
|
181
|
+
// Failsafe: never hold the mic forever. Also aborts the take the
|
|
182
|
+
// moment the user switches away from the session it belongs to
|
|
183
|
+
// (checked every 500 ms).
|
|
184
|
+
const timer = setInterval(() => {
|
|
185
|
+
if (finished) return
|
|
186
|
+
if (shouldAbort?.()) {
|
|
187
|
+
finished = true
|
|
188
|
+
stopFn = () => {}
|
|
189
|
+
clearInterval(timer)
|
|
190
|
+
try {
|
|
191
|
+
child.kill("SIGKILL")
|
|
192
|
+
} catch {
|
|
193
|
+
// already exited
|
|
194
|
+
}
|
|
195
|
+
log("capture aborted — session switched mid-recording")
|
|
196
|
+
reject(new Error("take-aborted"))
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
if (state.elapsed >= opts.maxMs + 5000) finish("watchdog")
|
|
200
|
+
}, 500)
|
|
201
|
+
stopFn = () => finish("manual-stop")
|
|
202
|
+
})
|
|
203
|
+
return { done, stop: () => stopFn() }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function transcribe(wav: Uint8Array, opts: ReturnType<typeof mergeOptions>): Promise<string> {
|
|
207
|
+
const form = new FormData()
|
|
208
|
+
form.append("model", opts.model)
|
|
209
|
+
// Copy into a fresh ArrayBuffer-backed view: BlobPart rejects SharedArrayBuffer views.
|
|
210
|
+
form.append("file", new Blob([Buffer.from(wav)], { type: "audio/wav" }), "utterance.wav")
|
|
211
|
+
const response = await fetch(sttEndpoint(opts.stt), { method: "POST", body: form })
|
|
212
|
+
if (!response.ok) throw new Error(`STT HTTP ${response.status} ${response.statusText}`)
|
|
213
|
+
const body = (await response.json()) as { text?: string }
|
|
214
|
+
return (body.text ?? "").trim()
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Display width of text — cursorOffset is measured in width units (host does
|
|
218
|
+
* the same via Bun.stringWidth); plain UTF-16 length is the ASCII fallback. */
|
|
219
|
+
function displayWidth(text: string): number {
|
|
220
|
+
const bun = (globalThis as { Bun?: { stringWidth?: (s: string) => number } }).Bun
|
|
221
|
+
return bun?.stringWidth ? bun.stringWidth(text) : text.length
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// One session's draft.
|
|
225
|
+
interface DraftState {
|
|
226
|
+
text: string
|
|
227
|
+
takes: number
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const EMPTY_DRAFT: DraftState = { text: "", takes: 0 }
|
|
231
|
+
|
|
232
|
+
export default Plugin.define({
|
|
233
|
+
id: "voice",
|
|
234
|
+
setup(ctx) {
|
|
235
|
+
// TODO: wire debug to plugin options once discovery supports them.
|
|
236
|
+
const opts = { ...mergeOptions(ctx.options as VoiceOptions | undefined), debug: true }
|
|
237
|
+
const log = (m: string) => dbg(opts.debug, m)
|
|
238
|
+
|
|
239
|
+
// --- plugin-scope state -------------------------------------------------
|
|
240
|
+
// Per-session drafts, kept at setup scope: switching sessions remounts
|
|
241
|
+
// the prompt footer (render-scoped state would reset), and each session
|
|
242
|
+
// must keep its own draft anyway. Keyed by sessionID.
|
|
243
|
+
const [phase, setPhase] = createSignal<Phase>("idle")
|
|
244
|
+
const [drafts, setDrafts] = createSignal<Record<string, DraftState>>({})
|
|
245
|
+
const draftOf = (sessionID: string): DraftState => drafts()[sessionID] ?? EMPTY_DRAFT
|
|
246
|
+
const putDraft = (sessionID: string, patch: Partial<DraftState>): void => {
|
|
247
|
+
setDrafts({ ...drafts(), [sessionID]: { ...draftOf(sessionID), ...patch } })
|
|
248
|
+
}
|
|
249
|
+
// --- server health ------------------------------------------------------
|
|
250
|
+
// Heartbeat for the local STT server behind the footer mic dot. Read-only
|
|
251
|
+
// GETs, fire-and-forget, never blocking dictation. Runs on load and every
|
|
252
|
+
// 30 s; dictate attempts and STT failures force an out-of-band refresh.
|
|
253
|
+
const localServer = isLocalEndpoint(opts.stt)
|
|
254
|
+
const [server, setServer] = createSignal<ServerHealth>("unknown")
|
|
255
|
+
let lastHealthCheck = 0
|
|
256
|
+
const checkHealth = async (force = false): Promise<void> => {
|
|
257
|
+
if (!localServer) return
|
|
258
|
+
if (!force && Date.now() - lastHealthCheck < HEALTH_DEBOUNCE_MS) return
|
|
259
|
+
lastHealthCheck = Date.now()
|
|
260
|
+
for (const url of healthEndpoints(opts.stt)) {
|
|
261
|
+
try {
|
|
262
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) })
|
|
263
|
+
if (response.ok) {
|
|
264
|
+
if (server() !== "up") log("stt server up")
|
|
265
|
+
setServer("up")
|
|
266
|
+
return
|
|
267
|
+
}
|
|
268
|
+
} catch {
|
|
269
|
+
// try the next candidate
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (server() !== "down") log("stt server down")
|
|
273
|
+
setServer("down")
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
let heartbeat: ReturnType<typeof setInterval> | undefined
|
|
277
|
+
if (localServer) {
|
|
278
|
+
void checkHealth(true) // no "unknown" dot lingering after load
|
|
279
|
+
heartbeat = setInterval(() => void checkHealth(), HEALTH_POLL_MS)
|
|
280
|
+
}
|
|
281
|
+
// Toggle state: f9 while recording stops the take instead of starting one.
|
|
282
|
+
let stopActive: (() => void) | null = null
|
|
283
|
+
|
|
284
|
+
interface ActiveEditor {
|
|
285
|
+
getContent: () => string
|
|
286
|
+
/** Session whose draft this editor edits. */
|
|
287
|
+
sessionID: string
|
|
288
|
+
}
|
|
289
|
+
let active: ActiveEditor | null = null
|
|
290
|
+
|
|
291
|
+
const persistActive = (): void => {
|
|
292
|
+
if (!active) return
|
|
293
|
+
const sessionID = active.sessionID
|
|
294
|
+
const content = active.getContent()
|
|
295
|
+
if (content.trim()) {
|
|
296
|
+
putDraft(sessionID, { text: content })
|
|
297
|
+
log(`persisted editor edits session=${sessionID} chars=${content.length}`)
|
|
298
|
+
} else {
|
|
299
|
+
putDraft(sessionID, { text: "", takes: 0 })
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const dictate = async () => {
|
|
304
|
+
// Second f9 while recording: stop the take, transcription follows.
|
|
305
|
+
if (stopActive) {
|
|
306
|
+
ctx.ui.toast.show({ message: "Stopped — transcribing…", variant: "info", duration: 2000 })
|
|
307
|
+
log("manual stop")
|
|
308
|
+
const stop = stopActive
|
|
309
|
+
stopActive = null
|
|
310
|
+
stop()
|
|
311
|
+
return
|
|
312
|
+
}
|
|
313
|
+
// Never overlap a second recording onto an in-flight transcription.
|
|
314
|
+
if (phase() === "transcribing") {
|
|
315
|
+
ctx.ui.toast.show({ message: "Still transcribing — one moment.", variant: "warning", duration: 2000 })
|
|
316
|
+
return
|
|
317
|
+
}
|
|
318
|
+
const route = ctx.ui.router.current()
|
|
319
|
+
let sessionID: string
|
|
320
|
+
if (route.type === "session") {
|
|
321
|
+
sessionID = route.sessionID
|
|
322
|
+
} else if (route.type === "home") {
|
|
323
|
+
// A brand-new blank session renders the home/launch route.
|
|
324
|
+
// Dictating there starts the session, exactly like typing a
|
|
325
|
+
// prompt on the launch view would.
|
|
326
|
+
try {
|
|
327
|
+
const created = await ctx.client.session.create({})
|
|
328
|
+
sessionID = created.id
|
|
329
|
+
ctx.ui.router.navigate({ type: "session", sessionID })
|
|
330
|
+
log(`created session ${sessionID} for dictation`)
|
|
331
|
+
} catch (error) {
|
|
332
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
333
|
+
log(`session create failed: ${message}`)
|
|
334
|
+
ctx.ui.toast.show({
|
|
335
|
+
title: "Voice",
|
|
336
|
+
message: `Could not start a session: ${message}`,
|
|
337
|
+
variant: "error",
|
|
338
|
+
})
|
|
339
|
+
return
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
ctx.ui.toast.show({ message: "Open a session first, then dictate.", variant: "warning" })
|
|
343
|
+
return
|
|
344
|
+
}
|
|
345
|
+
// New take from inside the editor: persist edits, close it, record.
|
|
346
|
+
if (active) {
|
|
347
|
+
persistActive()
|
|
348
|
+
active = null
|
|
349
|
+
ctx.ui.dialog.clear()
|
|
350
|
+
}
|
|
351
|
+
// Freshest possible dot before recording (debounced: free when recent).
|
|
352
|
+
await checkHealth()
|
|
353
|
+
setPhase("recording")
|
|
354
|
+
ctx.ui.toast.show({
|
|
355
|
+
message: opts.toggle ? "Recording… f9 to stop." : "Listening… speak now.",
|
|
356
|
+
variant: "info",
|
|
357
|
+
duration: 2000,
|
|
358
|
+
})
|
|
359
|
+
log("dictate started")
|
|
360
|
+
// Discard the take if the user switches away from its session: checked
|
|
361
|
+
// while recording (500 ms ticks), after stop (before STT), and after
|
|
362
|
+
// STT. Only the take is discarded — every draft stays untouched.
|
|
363
|
+
const switchedAway = (): boolean => {
|
|
364
|
+
const r = ctx.ui.router.current()
|
|
365
|
+
return r.type !== "session" || r.sessionID !== sessionID
|
|
366
|
+
}
|
|
367
|
+
const cap = capture(opts, log, switchedAway)
|
|
368
|
+
stopActive = cap.stop
|
|
369
|
+
let audio: Capture
|
|
370
|
+
try {
|
|
371
|
+
audio = await cap.done
|
|
372
|
+
} catch (error) {
|
|
373
|
+
stopActive = null
|
|
374
|
+
setPhase("idle")
|
|
375
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
376
|
+
if (message === "take-aborted") {
|
|
377
|
+
ctx.ui.toast.show({
|
|
378
|
+
message: "Take discarded — you switched chats mid-recording.",
|
|
379
|
+
variant: "warning",
|
|
380
|
+
duration: 3000,
|
|
381
|
+
})
|
|
382
|
+
return
|
|
383
|
+
}
|
|
384
|
+
log(`capture failed: ${message}`)
|
|
385
|
+
ctx.ui.toast.show({ title: "Voice", message: `Mic failed: ${message}`, variant: "error" })
|
|
386
|
+
return
|
|
387
|
+
}
|
|
388
|
+
stopActive = null
|
|
389
|
+
if (!audio.hadSpeech) {
|
|
390
|
+
setPhase("idle")
|
|
391
|
+
ctx.ui.toast.show({ message: "Heard nothing — try again.", variant: "warning" })
|
|
392
|
+
return
|
|
393
|
+
}
|
|
394
|
+
// Checkpoint: switched between stop and transcription — skip the STT
|
|
395
|
+
// upload entirely.
|
|
396
|
+
if (switchedAway()) {
|
|
397
|
+
setPhase("idle")
|
|
398
|
+
log("discarded take — session switched before transcription")
|
|
399
|
+
ctx.ui.toast.show({ message: "Take discarded — you switched chats.", variant: "warning", duration: 3000 })
|
|
400
|
+
return
|
|
401
|
+
}
|
|
402
|
+
setPhase("transcribing")
|
|
403
|
+
const sttStarted = Date.now()
|
|
404
|
+
let text: string
|
|
405
|
+
try {
|
|
406
|
+
text = await transcribe(audio.wav, opts)
|
|
407
|
+
} catch (error) {
|
|
408
|
+
setPhase("idle")
|
|
409
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
410
|
+
log(`transcribe failed: ${message}`)
|
|
411
|
+
// A green dot must never survive a failed take: force the re-check
|
|
412
|
+
// so the footer tells the truth within a second.
|
|
413
|
+
await checkHealth(true)
|
|
414
|
+
ctx.ui.toast.show({
|
|
415
|
+
title: "Voice",
|
|
416
|
+
message: existsSync(installedSkillPath())
|
|
417
|
+
? `STT failed (${message}) — run @voice-check to restart the server.`
|
|
418
|
+
: `STT failed (${message}) — run /voice-setup once, then @voice-check.`,
|
|
419
|
+
variant: "error",
|
|
420
|
+
})
|
|
421
|
+
return
|
|
422
|
+
}
|
|
423
|
+
if (!text) {
|
|
424
|
+
setPhase("idle")
|
|
425
|
+
ctx.ui.toast.show({ message: "Empty transcription — try again.", variant: "warning" })
|
|
426
|
+
return
|
|
427
|
+
}
|
|
428
|
+
log(`transcript="${text}"`)
|
|
429
|
+
log(`timing audio=${Math.round(audio.voicedMs)}ms stt=${Date.now() - sttStarted}ms bytes=${audio.wav.length}`)
|
|
430
|
+
// Checkpoint: switched during transcription — the transcript is
|
|
431
|
+
// discarded, never appended to any draft.
|
|
432
|
+
if (switchedAway()) {
|
|
433
|
+
setPhase("idle")
|
|
434
|
+
log("discarded transcript — session switched during transcription")
|
|
435
|
+
ctx.ui.toast.show({ message: "Take discarded — you switched chats.", variant: "warning", duration: 3000 })
|
|
436
|
+
return
|
|
437
|
+
}
|
|
438
|
+
setPhase("idle")
|
|
439
|
+
if (!opts.accumulate) {
|
|
440
|
+
openEditor(sessionID, text, false)
|
|
441
|
+
return
|
|
442
|
+
}
|
|
443
|
+
const current = draftOf(sessionID)
|
|
444
|
+
const merged = current.text ? `${current.text}\n\n${text}` : text
|
|
445
|
+
putDraft(sessionID, { text: merged, takes: current.takes + 1 })
|
|
446
|
+
log(`draft session=${sessionID} take=${current.takes + 1} chars=${merged.length}`)
|
|
447
|
+
openEditor(sessionID, merged, true)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const openSendDialog = () => {
|
|
451
|
+
const route = ctx.ui.router.current()
|
|
452
|
+
if (route.type !== "session") {
|
|
453
|
+
ctx.ui.toast.show({ message: "Open a session first.", variant: "warning" })
|
|
454
|
+
return
|
|
455
|
+
}
|
|
456
|
+
// Fold in edits from an editor the host may have closed itself.
|
|
457
|
+
persistActive()
|
|
458
|
+
const sessionID = route.sessionID
|
|
459
|
+
// Open the editor even when this session's draft is empty: it doubles
|
|
460
|
+
// as a place to compose by hand and then send.
|
|
461
|
+
openEditor(sessionID, draftOf(sessionID).text, true)
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// F12 wipes the draft of the session being viewed.
|
|
465
|
+
const clearDraft = async () => {
|
|
466
|
+
const hadEditor = active !== null
|
|
467
|
+
const editorSession = active?.sessionID
|
|
468
|
+
if (active) {
|
|
469
|
+
persistActive()
|
|
470
|
+
active = null
|
|
471
|
+
ctx.ui.dialog.clear()
|
|
472
|
+
}
|
|
473
|
+
const route = ctx.ui.router.current()
|
|
474
|
+
if (route.type !== "session") {
|
|
475
|
+
ctx.ui.toast.show({ message: "Open a session first.", variant: "warning" })
|
|
476
|
+
return
|
|
477
|
+
}
|
|
478
|
+
const sessionID = route.sessionID
|
|
479
|
+
const current = draftOf(sessionID)
|
|
480
|
+
if (!current.text.trim()) {
|
|
481
|
+
ctx.ui.toast.show({ message: "Draft is already empty.", variant: "info" })
|
|
482
|
+
return
|
|
483
|
+
}
|
|
484
|
+
const confirmed = await ctx.ui.dialog.confirm({
|
|
485
|
+
title: "Wipe voice draft?",
|
|
486
|
+
message: `Discard this session's draft (${current.text.length} chars)? This cannot be undone.`,
|
|
487
|
+
label: { confirm: "Wipe draft", cancel: "Keep it" },
|
|
488
|
+
})
|
|
489
|
+
if (confirmed === true) {
|
|
490
|
+
putDraft(sessionID, { text: "", takes: 0 })
|
|
491
|
+
ctx.ui.dialog.clear()
|
|
492
|
+
ctx.ui.toast.show({ message: "Voice draft wiped.", variant: "info" })
|
|
493
|
+
return
|
|
494
|
+
}
|
|
495
|
+
// Kept: if we closed this session's editor for the confirm, reopen it
|
|
496
|
+
// exactly as it was (edits were persisted before the confirm appeared).
|
|
497
|
+
if (hadEditor && editorSession === sessionID) {
|
|
498
|
+
openEditor(sessionID, draftOf(sessionID).text, true)
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// --- editor --------------------------------------------------------------
|
|
503
|
+
// One dialog, idempotent. Opening always clears any previous dialog and
|
|
504
|
+
// shows the text with the cursor at the end; the textarea is height-
|
|
505
|
+
// capped like the host's own prompt, so long drafts scroll internally
|
|
506
|
+
// and the cursor stays visible.
|
|
507
|
+
const openEditor = (sessionID: string, initial: string, persistDraft: boolean) => {
|
|
508
|
+
let edited = initial
|
|
509
|
+
let sent = false
|
|
510
|
+
let cursorPlaced = false
|
|
511
|
+
let area: { plainText?: string; cursorOffset?: number } | undefined
|
|
512
|
+
|
|
513
|
+
// Live-tracked content: prefer the renderable itself, fall back to
|
|
514
|
+
// the onContentChange mirror (survives renderable destruction).
|
|
515
|
+
const getContent = (): string => {
|
|
516
|
+
try {
|
|
517
|
+
const t = area?.plainText
|
|
518
|
+
if (typeof t === "string") return t
|
|
519
|
+
} catch {
|
|
520
|
+
// destroyed renderable
|
|
521
|
+
}
|
|
522
|
+
return edited
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
const self: ActiveEditor = { getContent, sessionID }
|
|
526
|
+
const header = persistDraft
|
|
527
|
+
? "🎤 Voice draft — edit · Ctrl+Enter send · Esc keep edits"
|
|
528
|
+
: "🎤 Dictation — edit · Ctrl+Enter send · Esc cancel"
|
|
529
|
+
|
|
530
|
+
const finish = async (value: string) => {
|
|
531
|
+
if (sent || active !== self) return
|
|
532
|
+
const text = value.trim()
|
|
533
|
+
if (!text) {
|
|
534
|
+
cancel()
|
|
535
|
+
return
|
|
536
|
+
}
|
|
537
|
+
// Refuse to deliver to a session other than the one visible now.
|
|
538
|
+
const route = ctx.ui.router.current()
|
|
539
|
+
if (route.type !== "session" || route.sessionID !== sessionID) {
|
|
540
|
+
ctx.ui.toast.show({
|
|
541
|
+
title: "Voice",
|
|
542
|
+
message: "Session changed — not sent. Draft kept.",
|
|
543
|
+
variant: "warning",
|
|
544
|
+
})
|
|
545
|
+
return
|
|
546
|
+
}
|
|
547
|
+
sent = true
|
|
548
|
+
if (persistDraft) putDraft(sessionID, { text }) // a failed send must not lose the latest text
|
|
549
|
+
try {
|
|
550
|
+
await ctx.client.session.prompt({ sessionID, text })
|
|
551
|
+
} catch (error) {
|
|
552
|
+
sent = false
|
|
553
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
554
|
+
log(`send failed: ${message}`)
|
|
555
|
+
ctx.ui.toast.show({
|
|
556
|
+
title: "Voice",
|
|
557
|
+
message: `Send failed: ${message} — draft kept, Ctrl+Enter to retry.`,
|
|
558
|
+
variant: "error",
|
|
559
|
+
})
|
|
560
|
+
return // editor stays open; retry in place
|
|
561
|
+
}
|
|
562
|
+
active = null
|
|
563
|
+
ctx.ui.dialog.clear()
|
|
564
|
+
if (persistDraft) putDraft(sessionID, { text: "", takes: 0 })
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const cancel = () => {
|
|
568
|
+
if (active !== self) return
|
|
569
|
+
if (persistDraft) persistActive()
|
|
570
|
+
active = null
|
|
571
|
+
ctx.ui.dialog.clear()
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
active = self
|
|
575
|
+
ctx.ui.dialog.clear() // never stack editors
|
|
576
|
+
ctx.ui.dialog.set({ size: "large", centered: true })
|
|
577
|
+
// The host's own prompt caps its textarea the same way: bounded
|
|
578
|
+
// height makes the editor scroll internally and keep the cursor
|
|
579
|
+
// visible.
|
|
580
|
+
const termRows = (process.stdout as { rows?: number }).rows ?? 40
|
|
581
|
+
const maxHeight = Math.max(6, Math.floor(termRows * 0.5))
|
|
582
|
+
ctx.ui.dialog.show(
|
|
583
|
+
() => (
|
|
584
|
+
<box flexDirection="column" gap={1} padding={1}>
|
|
585
|
+
<text>{header}</text>
|
|
586
|
+
<textarea
|
|
587
|
+
width="100%"
|
|
588
|
+
minHeight={1}
|
|
589
|
+
maxHeight={maxHeight}
|
|
590
|
+
initialValue={initial}
|
|
591
|
+
focused
|
|
592
|
+
keyBindings={[{ name: "return", ctrl: true, action: "submit" }]}
|
|
593
|
+
ref={(el: unknown) => {
|
|
594
|
+
area = el as { plainText?: string; cursorOffset?: number }
|
|
595
|
+
// Cursor at the end: new takes land there and edits
|
|
596
|
+
// usually continue at the tail of the draft. Deferred —
|
|
597
|
+
// the host's own prompt mutates the textarea post-mount
|
|
598
|
+
// the same way (the initialValue prop sync would otherwise
|
|
599
|
+
// run after an immediate set and reset the cursor).
|
|
600
|
+
if (!cursorPlaced) {
|
|
601
|
+
cursorPlaced = true
|
|
602
|
+
setTimeout(() => {
|
|
603
|
+
const el2 = area
|
|
604
|
+
if (!el2) return
|
|
605
|
+
try {
|
|
606
|
+
el2.cursorOffset = displayWidth(initial)
|
|
607
|
+
} catch {
|
|
608
|
+
// optional nicety, never fatal
|
|
609
|
+
}
|
|
610
|
+
}, 0)
|
|
611
|
+
}
|
|
612
|
+
}}
|
|
613
|
+
onContentChange={(value: unknown) => {
|
|
614
|
+
if (typeof value === "string") edited = value
|
|
615
|
+
}}
|
|
616
|
+
onSubmit={() => finish(getContent())}
|
|
617
|
+
onKeyDown={(event: unknown) => {
|
|
618
|
+
const name = (event as { name?: string } | null)?.name
|
|
619
|
+
if (name === "escape") cancel()
|
|
620
|
+
}}
|
|
621
|
+
/>
|
|
622
|
+
</box>
|
|
623
|
+
),
|
|
624
|
+
() => {
|
|
625
|
+
// The host closed the dialog itself (Esc handled at the dialog
|
|
626
|
+
// level): fold the edits into the draft so nothing is lost.
|
|
627
|
+
if (active !== self) return
|
|
628
|
+
if (persistDraft) persistActive()
|
|
629
|
+
active = null
|
|
630
|
+
},
|
|
631
|
+
)
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// One footer slot owns the visible parts: the live mic indicator plus
|
|
635
|
+
// the global keymap (layers need a component owner, the slot provides
|
|
636
|
+
// it). All state lives at setup scope above, so slot remounts (session
|
|
637
|
+
// switches) never lose drafts; the take count shown is the visible
|
|
638
|
+
// session's own.
|
|
639
|
+
// The slot's own cleanup plus the heartbeat: composed so a plugin
|
|
640
|
+
// reload removes the footer/keymap AND stops the timer (no leaks).
|
|
641
|
+
const releaseSlot = ctx.ui.slot({
|
|
642
|
+
append: "prompt.footer.status",
|
|
643
|
+
render: (input) => {
|
|
644
|
+
ctx.keymap.layer(() => ({
|
|
645
|
+
mode: "global",
|
|
646
|
+
commands: [
|
|
647
|
+
{
|
|
648
|
+
id: "voice.dictate",
|
|
649
|
+
title: "Dictate prompt (voice)",
|
|
650
|
+
description: "Record one utterance, transcribe locally, add to the draft.",
|
|
651
|
+
group: "Voice",
|
|
652
|
+
bind: "f9",
|
|
653
|
+
palette: true,
|
|
654
|
+
slash: { name: "dictate" },
|
|
655
|
+
run: () => dictate(),
|
|
656
|
+
},
|
|
657
|
+
{
|
|
658
|
+
id: "voice.send",
|
|
659
|
+
title: "Review voice draft",
|
|
660
|
+
description: "Open the session's draft editor (empty or not). Ctrl+Enter sends, Esc keeps.",
|
|
661
|
+
group: "Voice",
|
|
662
|
+
bind: "f10",
|
|
663
|
+
palette: true,
|
|
664
|
+
slash: { name: "voice-send" },
|
|
665
|
+
run: () => openSendDialog(),
|
|
666
|
+
},
|
|
667
|
+
{
|
|
668
|
+
id: "voice.clear",
|
|
669
|
+
title: "Wipe voice draft",
|
|
670
|
+
description: "Discard this session's draft (asks for confirmation).",
|
|
671
|
+
group: "Voice",
|
|
672
|
+
bind: "f12",
|
|
673
|
+
palette: true,
|
|
674
|
+
slash: { name: "voice-clear" },
|
|
675
|
+
run: () => clearDraft(),
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
id: "voice.setup",
|
|
679
|
+
title: "Install voice-check recovery skill",
|
|
680
|
+
description: "One-time opt-in: install the voice-check STT recovery skill globally.",
|
|
681
|
+
group: "Voice",
|
|
682
|
+
palette: true,
|
|
683
|
+
slash: { name: "voice-setup" },
|
|
684
|
+
run: () => {
|
|
685
|
+
const result = installSkill(log)
|
|
686
|
+
if (result === "installed") {
|
|
687
|
+
ctx.ui.toast.show({
|
|
688
|
+
title: "Voice",
|
|
689
|
+
message: "voice-check skill installed — type @voice-check or say 'voice is dead' any time.",
|
|
690
|
+
variant: "success",
|
|
691
|
+
duration: 5000,
|
|
692
|
+
})
|
|
693
|
+
} else if (result === "exists") {
|
|
694
|
+
ctx.ui.toast.show({ message: "voice-check skill is already installed.", variant: "info" })
|
|
695
|
+
} else {
|
|
696
|
+
ctx.ui.toast.show({
|
|
697
|
+
title: "Voice",
|
|
698
|
+
message: "Could not install the skill — see docs/STT.md for manual recovery.",
|
|
699
|
+
variant: "error",
|
|
700
|
+
})
|
|
701
|
+
}
|
|
702
|
+
},
|
|
703
|
+
},
|
|
704
|
+
],
|
|
705
|
+
bindings: ["voice.dictate", "voice.send", "voice.clear", "voice.setup"],
|
|
706
|
+
}))
|
|
707
|
+
|
|
708
|
+
const label = () => {
|
|
709
|
+
const p = phase()
|
|
710
|
+
if (p === "recording") return "🔴 REC f9"
|
|
711
|
+
if (p === "transcribing") return "🟡 …"
|
|
712
|
+
const sid = input.sessionID
|
|
713
|
+
const n = sid ? (drafts()[sid]?.takes ?? 0) : 0
|
|
714
|
+
const mic = n > 0 ? `🎤·${n} f9` : "🎤 f9"
|
|
715
|
+
// Health dot only in idle (phases own the footer during a take)
|
|
716
|
+
// and only for local servers; unknown renders dotless.
|
|
717
|
+
if (!localServer) return mic
|
|
718
|
+
const s = server()
|
|
719
|
+
if (s === "up") return `🟢${mic}`
|
|
720
|
+
if (s === "down") return `🔴${mic}`
|
|
721
|
+
return mic
|
|
722
|
+
}
|
|
723
|
+
return <text>{label()}</text>
|
|
724
|
+
},
|
|
725
|
+
})
|
|
726
|
+
|
|
727
|
+
return () => {
|
|
728
|
+
if (heartbeat !== undefined) clearInterval(heartbeat)
|
|
729
|
+
releaseSlot()
|
|
730
|
+
}
|
|
731
|
+
},
|
|
732
|
+
})
|