@bojackduy/opencode-voice 0.11.4 → 0.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -114,7 +114,8 @@ export function createConversationStatus(
114
114
  function redraw() {
115
115
  const segMap = new Map(segments().map((seg) => [seg.key, seg.text]));
116
116
  let changed = false;
117
- for (const [key, text] of segMap) {
117
+ for (const key of ["status", "cancel", "send", "barge", "stop", "talk", "end", "pause"]) {
118
+ const text = segMap.has(key) ? segMap.get(key) : "";
118
119
  if (lastTexts.get(key) !== text) {
119
120
  changed = true;
120
121
  break;
@@ -126,7 +127,8 @@ export function createConversationStatus(
126
127
  }
127
128
  return;
128
129
  }
129
- for (const [key, text] of segMap) {
130
+ for (const key of ["status", "cancel", "send", "barge", "stop", "talk", "end", "pause"]) {
131
+ const text = segMap.has(key) ? segMap.get(key) : "";
130
132
  lastTexts.set(key, text);
131
133
  for (const entry of nodes) {
132
134
  if (entry.key !== key || entry.node.isDestroyed) continue;
@@ -218,9 +220,12 @@ export function createConversationStatus(
218
220
  const action = segMap.has(key)
219
221
  ? (segments().find((s) => s.key === key)?.action ?? null)
220
222
  : null;
221
- if (action) {
223
+ if (true) {
222
224
  try {
223
- setProp(node, "onMouseUp", () => press(action));
225
+ setProp(node, "onMouseUp", () => {
226
+ const seg = segments().find((s) => s.key === key);
227
+ if (seg?.action) press(seg.action);
228
+ });
224
229
  } catch {}
225
230
  }
226
231
  const entry = { key, node };
@@ -23,13 +23,11 @@
23
23
  // or the global /voice-cancel. Empty or failed turns pause instead of
24
24
  // auto-recording, so the key never surprises.
25
25
  //
26
- // Latency: while waiting, assistant text deltas are spoken sentence by
27
- // sentence (local cleanup, no LLM), so the answer starts before the turn
28
- // completes. Replies with nothing streamable fall back to the full
29
- // LLM-narrated speak.
26
+ // Latency vs. honesty: the turn is voiced only after the whole execution
27
+ // (tool calls included) completes, so what is read aloud is the answer and
28
+ // never mid-turn commentary.
30
29
 
31
30
  import { clearProcessingToast, showProcessingToast } from "./stt.js";
32
- import { isSpeakableSentence, localSpeechCleanup, splitSpokenSentences } from "./tts.js";
33
31
 
34
32
  export const DEFAULT_STOP_PHRASES = [
35
33
  "stop",
@@ -105,28 +103,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
105
103
  let active = false;
106
104
  let phase = "idle"; // idle | recording | processing | waiting | speaking | paused
107
105
 
108
- // Reply streaming state: text deltas heard while the agent works are spoken
109
- // sentence by sentence through a serial queue, so the answer starts before
110
- // the turn completes and every sentence gets its own vi/en voice.
111
-
112
- let streamBuffer = "";
113
- let streamSpokenChars = 0;
106
+ // Event subscriptions owned by the current wait/probe set. Nothing is
107
+ // spoken from live deltas: the agent's commentary during tool calls is not
108
+ // the answer, so only the completed turn is voiced (see speakReply).
114
109
  let streamUnsubs = [];
115
- let speakTail = Promise.resolve();
116
110
  let speakGen = 0;
117
111
 
118
- function queueSpeak(sentence) {
119
- const g = speakGen;
120
- speakTail = speakTail.then(() => {
121
- // Speaking must also drain: finishTurnFlow flips to speaking before
122
- // flushing the tail, so a waiting-only guard would skip queued
123
- // sentences/final partials under real TTS latency. speakGen still
124
- // invalidates on pause/stop.
125
- if (g !== speakGen || !active || (phase !== "waiting" && phase !== "speaking")) return null;
126
- return tts.speak(sentence);
127
- });
128
- return speakTail;
129
- }
130
112
  let turn = 0;
131
113
  let consecErrors = 0;
132
114
  let generation = 0;
@@ -139,6 +121,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
139
121
  // v2 persistent status line: phase-aware buttons. v1 has no surface, so
140
122
  // these are no-ops there (api.conversationStatus is undefined).
141
123
  const convStatus = api.conversationStatus || null;
124
+ const clog = (m) => {
125
+ try {
126
+ stt.log?.(m);
127
+ } catch {}
128
+ };
142
129
  const setConvPhase = api.setConvPhase || (() => {});
143
130
  const setConvActions = api.setConvActions || (() => {});
144
131
  function convBegin() {
@@ -188,18 +175,72 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
188
175
  if (stt.isRecording() || stt.isProcessing() || stt.isStreaming?.()) return;
189
176
  turn += 1;
190
177
  phase = "recording";
191
- logger?.log("VOICE", `Conversation turn ${turn} listening`, "debug");
192
- // Hands-free: auto-finalize after N seconds of silence so the user
193
- // never has to press a key to submit.
194
- stt.setSilenceAutoFinalize?.(1.5, () => {
195
- if (active && phase === "recording") {
196
- logger?.log("VOICE", "Conversation auto-finalize on silence", "debug");
197
- finishTurnFlow();
178
+ clog(`Conversation turn ${turn} listening`);
179
+ const streamingMode = (api?.kv?.get?.("stt.mode") ?? opts?.sttMode) === "streaming";
180
+ // Allow streaming STT during conversation for waveform + live transcript.
181
+ try {
182
+ stt.setConversationStreamingEnabled?.(true);
183
+ } catch {}
184
+ const autoEnd = api?.kv?.get?.("stt.autoEndOfSpeech") === true;
185
+ if (streamingMode) {
186
+ // Register the stop callback even when auto-end is off: the preview
187
+ // toggle can turn auto-end on mid-recording, and the STT timer must
188
+ // then finish this turn through the conversation loop instead of
189
+ // submitting the text as plain dictation (which skips the reply wait,
190
+ // the speech, and leaves the UI on "Listening").
191
+ const onStreamAutoStop = () => {
192
+ clog(`Conversation streaming auto-finalize callback active=${active} phase=${phase}`);
193
+ if (active && phase === "recording") {
194
+ clog("Conversation auto-finalize on silence (streaming)");
195
+ void finishTurnFlow();
196
+ }
197
+ };
198
+ stt.setConversationStreamingFinalize?.(autoEnd ? 2.5 : null, onStreamAutoStop);
199
+ const started = stt.startStreaming?.();
200
+ if (!started) {
201
+ // Fallback to batch recording.
202
+ stt.setConversationStreamingFinalize?.(null, null);
203
+ if (autoEnd) {
204
+ stt.setSilenceAutoFinalize?.(2.5, () => {
205
+ if (active && phase === "recording") {
206
+ clog("Conversation auto-finalize on silence");
207
+ finishTurnFlow();
208
+ }
209
+ });
210
+ } else {
211
+ stt.setSilenceAutoFinalize?.(null, null);
212
+ }
213
+ if (!stt.start()) {
214
+ phase = "idle";
215
+ stop("busy");
216
+ toast("STT busy, conversation off", "warning");
217
+ return;
218
+ }
198
219
  }
199
- });
220
+ } else {
221
+ if (autoEnd) {
222
+ stt.setSilenceAutoFinalize?.(2.5, () => {
223
+ if (active && phase === "recording") {
224
+ clog("Conversation auto-finalize on silence");
225
+ finishTurnFlow();
226
+ }
227
+ });
228
+ } else {
229
+ stt.setSilenceAutoFinalize?.(null, null);
230
+ }
231
+ if (!stt.start()) {
232
+ phase = "idle";
233
+ stop("busy");
234
+ toast("STT busy, conversation off", "warning");
235
+ return;
236
+ }
237
+ }
200
238
  convSetActions({
201
239
  cancel: () => {
202
240
  if (phase === "recording") {
241
+ try {
242
+ stt.cancelStreaming?.();
243
+ } catch {}
203
244
  stt.cancel();
204
245
  phase = "paused";
205
246
  convSetPhase("paused");
@@ -210,13 +251,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
210
251
  },
211
252
  });
212
253
  convSetPhase("recording");
213
- if (!stt.start()) {
214
- phase = "idle";
215
- stop("busy");
216
- toast("STT busy, conversation off", "warning");
217
- return;
218
- }
219
- toast(`Listening - speak, then pause ${1.5}s to send (or press ${keyLabel()})`);
254
+ toast(
255
+ autoEnd
256
+ ? `Listening - speak, then pause 2.5s to send (or press ${keyLabel()})`
257
+ : `Listening - press ${keyLabel()} to send`,
258
+ );
220
259
  }
221
260
 
222
261
  // The resting state: mic closed, one press to talk. Entered by start() and
@@ -239,8 +278,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
239
278
  // means "I want to talk now", so it must never exit silently.
240
279
 
241
280
  function bargeIn() {
242
- logger?.log("VOICE", "Conversation barge-in", "debug");
281
+ clog("Conversation barge-in");
243
282
  stt.setSilenceAutoFinalize?.(null, null);
283
+ stt.setConversationStreamingFinalize?.(null, null);
284
+ try {
285
+ stt.cancelStreaming?.();
286
+ } catch {}
244
287
  discardStreamAudio();
245
288
  clearProcessingToast();
246
289
  // Settle the pending reply wait as "stopped" so the abandoned answer is
@@ -277,6 +320,10 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
277
320
  generation += 1;
278
321
  phase = "idle";
279
322
  stt.setSilenceAutoFinalize?.(null, null);
323
+ stt.setConversationStreamingFinalize?.(null, null);
324
+ try {
325
+ stt.setConversationStreamingEnabled?.(false);
326
+ } catch {}
280
327
  if (waitCancel) {
281
328
  waitCancel("stopped");
282
329
  waitCancel = null;
@@ -284,72 +331,66 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
284
331
  endReplyStream();
285
332
  speakGen += 1;
286
333
  stt.setStopHint(null);
334
+ try {
335
+ stt.cancelStreaming?.();
336
+ } catch {}
287
337
  if (stt.isRecording() || stt.isProcessing()) stt.cancel();
288
338
  clearProcessingToast();
289
339
  tts.stop();
290
340
  tts.setConversationActive(false);
291
341
  convClear();
292
- logger?.log("VOICE", `Conversation stopped reason=${reason}`, "debug");
293
- }
294
-
295
- function startReplyStream(sessionID, turnStart) {
296
- streamBuffer = "";
297
- streamSpokenChars = 0;
298
- const unsub = api.event.on("message.part.delta", (event) => {
299
- try {
300
- if (!active || phase !== "waiting") return;
301
- const props = event.properties || {};
302
- const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
303
- if (typeof sessionID === "function" && !expectedID) return;
304
- if (expectedID && props.sessionID && props.sessionID !== expectedID) return;
305
- if (typeof props.delta !== "string" || !props.delta) return;
306
- if (props.field && props.field !== "text") return;
307
- const lookupID = props.sessionID || expectedID;
308
- const messages = api?.state?.session?.messages?.(lookupID) || [];
309
- const msg = messages.find((m) => m?.id === props.messageID);
310
- if (!msg || msg.role !== "assistant") return;
311
- if ((msg.time?.created || 0) < turnStart) return;
312
- const parts = api?.state?.part?.(props.messageID) || [];
313
- const part = parts.find((p) => p?.id === props.partID);
314
- if (part && part.type !== "text") return;
315
- streamBuffer += props.delta;
316
- flushStream(false);
317
- } catch {}
318
- });
319
- streamUnsubs.push(unsub);
320
- }
321
-
322
- function flushStream(final = false) {
323
- const { sentences, rest } = splitSpokenSentences(streamBuffer);
324
- const ready = final && rest.trim() ? [...sentences, rest] : sentences;
325
- if (ready.length > 0 && streamSpokenChars === 0) {
326
- showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
327
- }
328
- for (const sentence of ready) {
329
- if (!isSpeakableSentence(sentence)) continue;
330
- const cleaned = localSpeechCleanup(sentence);
331
- if (!cleaned) continue;
332
- queueSpeak(cleaned);
333
- streamSpokenChars += cleaned.length;
334
- }
335
- streamBuffer = final ? "" : rest;
342
+ clog(`Conversation stopped reason=${reason}`);
336
343
  }
337
344
 
338
345
  // Wait for the submitted turn to finish: idle, a permission/question gate,
339
346
  // or timeout. Resolves early when stop() cancels the wait. gates:false drops
340
347
  // the permission/question subscriptions, so a second wait survives the gate
341
348
  // the user is answering on screen and only ends on idle/timeout/stop.
342
-
343
- function waitForReply(sessionID, { gates = true } = {}) {
349
+ //
350
+ // Completion is a host fact, not a guess: v2 emits
351
+ // `session.execution.succeeded|failed|interrupted` only when the whole
352
+ // execution - tool calls included - ends. The message list is the fallback
353
+ // signal, and BOTH paths are gated on `baseline` (the message count taken
354
+ // before submit), so a finished assistant message from an earlier turn can
355
+ // never be mistaken for this turn's reply. That gate is the fix for the
356
+ // reply being read ~800ms early with zero assistant messages in it.
357
+
358
+ function waitForReply(sessionID, { gates = true, baseline = 0 } = {}) {
344
359
  let settled = false;
345
360
  let unsubs = [];
346
361
  let timer = null;
362
+ let pollTimer = null;
363
+ let settleTimer = null;
364
+ let currentExecutionID = null;
365
+ let sawExecutionStart = false;
366
+
367
+ const resolveSessionID = () => (typeof sessionID === "function" ? sessionID() : sessionID);
368
+ const baselineFor = (sid) => (typeof baseline === "function" ? baseline(sid) : baseline);
369
+
370
+ // Assistant messages that arrived AFTER submit, oldest first.
371
+ const newAssistants = () => {
372
+ try {
373
+ const sid = resolveSessionID();
374
+ const msgs = api?.state?.session?.messages?.(sid) || [];
375
+ return msgs.slice(baselineFor(sid)).filter((m) => m?.role === "assistant");
376
+ } catch {
377
+ return [];
378
+ }
379
+ };
380
+ const isTerminal = (finish) => !!finish && finish !== "tool-calls" && finish !== "tool_calls";
381
+ const lastFinish = () => {
382
+ const list = newAssistants();
383
+ return list.length ? list[list.length - 1]?.finish : null;
384
+ };
385
+ const turnFinished = () => isTerminal(lastFinish());
347
386
 
348
387
  let cancelFn = null;
349
388
  const done = (outcome) => {
350
389
  if (settled) return null;
351
390
  settled = true;
352
391
  if (timer) clearTimeout(timer);
392
+ if (settleTimer) clearInterval(settleTimer);
393
+ if (pollTimer) clearInterval(pollTimer);
353
394
  for (const unsub of unsubs) {
354
395
  try {
355
396
  unsub();
@@ -368,20 +409,93 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
368
409
  cancelFn = (outcome) => finish(outcome || "stopped");
369
410
  waitCancel = cancelFn;
370
411
 
412
+ // The host event can land a beat before the message cache does. Give the
413
+ // message at most 2s to appear, then settle anyway: the execution is
414
+ // over, and "no reply to speak" is a truthful outcome.
415
+ const settle = (outcome) => {
416
+ if (settled) return;
417
+ if (turnFinished()) {
418
+ finish(outcome);
419
+ return;
420
+ }
421
+ if (settleTimer) return;
422
+ const startedAt = Date.now();
423
+ settleTimer = setInterval(() => {
424
+ if (settled) return;
425
+ if (turnFinished() || Date.now() - startedAt > 2000) {
426
+ clog(`Conversation settled outcome=${outcome} finish=${lastFinish() ?? "none"}`);
427
+ finish(outcome);
428
+ }
429
+ }, 100);
430
+ if (typeof settleTimer.unref === "function") settleTimer.unref();
431
+ };
432
+
371
433
  const forSession = (props) => {
372
- const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
373
- if (typeof sessionID === "function" && !expectedID) return false;
374
- return !expectedID || !props?.sessionID || props.sessionID === expectedID;
434
+ const sid = props?.sessionID ?? props?.id ?? props?.session?.id;
435
+ const expectedID = resolveSessionID();
436
+ if (typeof sessionID === "function" && !expectedID) return true;
437
+ if (!expectedID) return true;
438
+ if (!sid) return true;
439
+ return sid === expectedID;
440
+ };
441
+
442
+ const executionIDOf = (props) =>
443
+ props?.executionID ?? props?.executionId ?? props?.id ?? props?.execution?.id;
444
+
445
+ const onExecutionEnd = (outcome) => (event) => {
446
+ const props = event?.properties || event?.data || {};
447
+ if (!forSession(props)) return;
448
+ const eid = executionIDOf(props);
449
+ if (currentExecutionID && eid && eid !== currentExecutionID) {
450
+ clog(
451
+ `Conversation execution ${outcome} ignored stale id=${eid} current=${currentExecutionID}`,
452
+ );
453
+ return;
454
+ }
455
+ // The host's execution events carry no execution id, so the guard is
456
+ // structural: a terminal event is only trusted once this wait has seen
457
+ // execution.started, or once this turn's own assistant message is
458
+ // there with a terminal finish. Otherwise it is a late event from the
459
+ // turn that barge-in interrupted.
460
+ if (!sawExecutionStart && !turnFinished()) {
461
+ clog(`Conversation execution ${outcome} ignored (no start, no new assistant)`);
462
+ return;
463
+ }
464
+ clog(`Conversation execution ${outcome} id=${eid ?? "none"}`);
465
+ settle(outcome);
375
466
  };
376
467
 
377
468
  unsubs = [
469
+ api.event.on("session.execution.started", (event) => {
470
+ const props = event?.properties || event?.data || {};
471
+ if (!forSession(props)) return;
472
+ const eid = executionIDOf(props);
473
+ if (eid) currentExecutionID = eid;
474
+ sawExecutionStart = true;
475
+ clog(`Conversation execution.started id=${eid ?? "none"}`);
476
+ }),
477
+ api.event.on("session.execution.succeeded", onExecutionEnd("idle")),
478
+ api.event.on("session.execution.failed", onExecutionEnd("idle")),
479
+ api.event.on("session.execution.interrupted", onExecutionEnd("stopped")),
480
+ // Legacy fallbacks, only trusted once this turn's assistant message
481
+ // actually exists with a terminal finish.
378
482
  api.event.on("session.idle", (event) => {
379
- if (forSession(event.properties)) finish("idle");
483
+ const props = event?.properties || event?.data || {};
484
+ if (!forSession(props)) return;
485
+ if (!turnFinished()) {
486
+ clog(`Conversation session.idle ignored finish=${lastFinish() ?? "none"}`);
487
+ return;
488
+ }
489
+ finish("idle");
380
490
  }),
381
491
  api.event.on("session.status", (event) => {
382
- if (event.properties?.status?.type === "idle" && forSession(event.properties)) {
383
- finish("idle");
492
+ const props = event?.properties || event?.data || {};
493
+ if (props?.status?.type !== "idle" || !forSession(props)) return;
494
+ if (!turnFinished()) {
495
+ clog(`Conversation session.status idle ignored finish=${lastFinish() ?? "none"}`);
496
+ return;
384
497
  }
498
+ finish("idle");
385
499
  }),
386
500
  ...(gates
387
501
  ? [
@@ -395,14 +509,28 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
395
509
  : []),
396
510
  ];
397
511
  timer = setTimeout(() => finish("timeout"), replyTimeoutMs);
512
+ // Last resort when the host emits no execution event at all.
513
+ pollTimer = setInterval(() => {
514
+ if (settled) return;
515
+ try {
516
+ const fr = lastFinish();
517
+ clog(`Conversation poll new=${newAssistants().length} finish=${fr ?? "none"}`);
518
+ if (isTerminal(fr)) settle("idle");
519
+ } catch {}
520
+ }, 700);
521
+ if (typeof pollTimer.unref === "function") pollTimer.unref();
398
522
  });
399
523
 
400
524
  return promise;
401
525
  }
402
526
 
403
527
  async function finishTurnFlow() {
528
+ clog(
529
+ `Conversation finishTurnFlow called active=${active} phase=${phase} streaming=${stt.isStreaming?.()}`,
530
+ );
404
531
  if (!active || phase !== "recording") return;
405
532
  stt.setSilenceAutoFinalize?.(null, null);
533
+ stt.setConversationStreamingFinalize?.(null, null);
406
534
  const myGen = generation;
407
535
  phase = "processing";
408
536
  convSetActions({
@@ -413,12 +541,25 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
413
541
  convSetPhase("processing");
414
542
  showProcessingToast("Transcribing...");
415
543
 
416
- const res = await stt.transcribeTurn();
544
+ let res = null;
545
+ if (stt.isStreaming?.()) {
546
+ try {
547
+ res = await stt.finalizeStreaming?.({ submit: false, insert: false });
548
+ } catch (e) {
549
+ res = { error: e };
550
+ }
551
+ } else {
552
+ res = await stt.transcribeTurn();
553
+ }
417
554
  if (!active || myGen !== generation) return;
418
555
 
556
+ if (!res) {
557
+ pauseForRetry(`Paused - press ${keyLabel()} to retry`);
558
+ return;
559
+ }
419
560
  if (res.error) {
420
561
  consecErrors += 1;
421
- logger?.log("VOICE", `Conversation turn error consec=${consecErrors}`, "warn");
562
+ clog(`Conversation turn error consec=${consecErrors}`);
422
563
  if (consecErrors >= 2) {
423
564
  stop("error");
424
565
  return;
@@ -426,7 +567,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
426
567
  pauseForRetry(`Paused - press ${keyLabel()} to retry`);
427
568
  return;
428
569
  }
429
- if (!res.text) {
570
+ if (!res.text || res.text.trim().length < 3) {
430
571
  consecErrors = 0;
431
572
  pauseForRetry(`No speech heard - press ${keyLabel()} to try again`);
432
573
  return;
@@ -440,7 +581,21 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
440
581
  return;
441
582
  }
442
583
 
443
- const turnStart = Date.now();
584
+ // Baseline: how many messages the session already holds BEFORE submit.
585
+ // Only messages after this index belong to this turn. A session created by
586
+ // the submit itself did not exist yet, so its baseline is 0.
587
+ const baselineSessionID = currentSessionID();
588
+ let baselineCount = 0;
589
+ if (baselineSessionID) {
590
+ try {
591
+ baselineCount = (api?.state?.session?.messages?.(baselineSessionID) || []).length;
592
+ } catch {
593
+ baselineCount = 0;
594
+ }
595
+ }
596
+ const baselineFor = (sid) => (sid && sid === baselineSessionID ? baselineCount : 0);
597
+ clog(`Conversation baseline sid=${baselineSessionID ?? "new"} count=${baselineCount}`);
598
+
444
599
  // v2 prompt dispatch may emit the entire reply before its promise returns.
445
600
  // Home creation navigates before prompting, so resolve the ID at event time.
446
601
  let earlyWait = null;
@@ -451,8 +606,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
451
606
  barge: () => bargeIn(),
452
607
  });
453
608
  convSetPhase("waiting");
454
- startReplyStream(currentSessionID, turnStart);
455
- earlyWait = waitForReply(currentSessionID);
609
+ earlyWait = waitForReply(currentSessionID, { baseline: baselineFor });
456
610
  }
457
611
  const submitted = await stt.submitTurnText(res.text);
458
612
  if (!active || myGen !== generation) return;
@@ -469,16 +623,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
469
623
  convSetPhase("waiting");
470
624
  const sessionID = currentSessionID();
471
625
  showProcessingToast(`Working - press ${keyLabel()} to talk`);
472
- if (!earlyWait) startReplyStream(sessionID, turnStart);
473
626
  const tWait = Date.now();
474
- const outcome = await (earlyWait || waitForReply(sessionID));
627
+ const outcome = await (earlyWait || waitForReply(sessionID, { baseline: baselineFor }));
475
628
  const waitMs = Date.now() - tWait;
476
629
  endReplyStream();
477
- logger?.log(
478
- "VOICE",
479
- `Conversation wait outcome=${outcome} waitMs=${waitMs} streamedChars=${streamSpokenChars}`,
480
- "debug",
481
- );
630
+ logger?.log("VOICE", `Conversation wait outcome=${outcome} waitMs=${waitMs}`, "debug");
482
631
  if (!active || myGen !== generation) return;
483
632
 
484
633
  if (outcome === "timeout") {
@@ -488,7 +637,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
488
637
  return;
489
638
  }
490
639
  if (outcome === "permission" || outcome === "question") {
491
- await answerGateThenSpeak(myGen, sessionID, turnStart, outcome);
640
+ await answerGateThenSpeak(myGen, sessionID, baselineFor, outcome);
492
641
  return;
493
642
  }
494
643
  if (outcome === "stopped") return;
@@ -501,7 +650,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
501
650
  // agent to resume so its post-answer reply is spoken aloud - before this,
502
651
  // endReplyStream() had already run and the answer was dropped in silence.
503
652
 
504
- async function answerGateThenSpeak(myGen, sessionID, turnStart, outcome) {
653
+ async function answerGateThenSpeak(myGen, sessionID, baseline, outcome) {
505
654
  discardStreamAudio();
506
655
  phase = "speaking";
507
656
  convSetActions({
@@ -540,8 +689,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
540
689
  });
541
690
  convSetPhase("waiting");
542
691
  showProcessingToast(`Answer on screen - press ${keyLabel()} to talk`);
543
- startReplyStream(sessionID, turnStart);
544
- const resumed = await waitForReply(sessionID, { gates: false });
692
+ const resumed = await waitForReply(sessionID, { gates: false, baseline });
545
693
  if (!active || myGen !== generation) return;
546
694
  if (resumed === "stopped") return;
547
695
  endReplyStream();
@@ -555,11 +703,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
555
703
  }
556
704
 
557
705
  // Speak whatever this turn produced, then rest paused with the mic OFF.
558
- // Streamed sentences are already queued, so only the tail needs flushing;
559
- // the full LLM-narrated speak is the fallback when nothing streamable
560
- // arrived. Returns early on any generation/phase change (barge-in, stop).
706
+ // Only the completed turn is voiced: live deltas during tool calls are
707
+ // commentary, not the answer, and speaking them reads half a thought aloud.
708
+ // Returns early on any generation/phase change (barge-in, stop).
561
709
 
562
710
  async function speakReply(myGen) {
711
+ clog(`Conversation speakReply entry myGen=${myGen} gen=${generation} phase=${phase}`);
563
712
  phase = "speaking";
564
713
  convSetActions({
565
714
  stop: () => {
@@ -580,21 +729,9 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
580
729
  });
581
730
  convSetPhase("speaking");
582
731
  showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
583
- if (streamSpokenChars > 0) {
584
- flushStream(true);
585
- await speakTail;
586
- clearProcessingToast();
587
- if (!active || myGen !== generation || phase !== "speaking") return;
588
- await delay(restartDelayMs);
589
- if (!active || myGen !== generation || phase !== "speaking") return;
590
- enterPaused();
591
- // Hands-free: auto-listen after the reply instead of waiting for a keypress.
592
- await delay(500);
593
- if (!active || myGen !== generation || phase !== "paused") return;
594
- beginTurn();
595
- return;
596
- }
732
+ clog("Conversation: calling tts.speakAssistantTurn");
597
733
  const spoken = await tts.speakAssistantTurn();
734
+ clog(`Conversation: speakAssistantTurn returned spoken=${spoken?.spoken}`);
598
735
  if (!active || myGen !== generation || phase !== "speaking") return;
599
736
  // Drop the sticky speaking status before the paused toast, or it keeps
600
737
  // re-announcing "interrupt" once the mic is closed again.
@@ -605,7 +742,8 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
605
742
  await delay(restartDelayMs);
606
743
  if (!active || myGen !== generation || phase !== "speaking") return;
607
744
  enterPaused();
608
- // Hands-free: auto-listen after the reply instead of waiting for a keypress.
745
+ if (!spoken.spoken) return;
746
+ // Hands-free: auto-listen after a successful reply.
609
747
  await delay(500);
610
748
  if (!active || myGen !== generation || phase !== "paused") return;
611
749
  beginTurn();
@@ -625,7 +763,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
625
763
 
626
764
  function pauseSpeech() {
627
765
  if (!active || phase !== "speaking") return;
628
- logger?.log("VOICE", "Conversation speech paused", "debug");
766
+ clog("Conversation speech paused");
629
767
  tts.stop();
630
768
  phase = "paused";
631
769
  toast(`Speech paused - press ${keyLabel()} to talk`);
@@ -700,7 +838,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
700
838
  convSetPhase("paused");
701
839
  // Recording toasts must name the key the user actually has bound.
702
840
  stt.setStopHint(kb("voice.conversation"));
703
- logger?.log("VOICE", "Conversation started", "debug");
841
+ clog("Conversation started");
704
842
  toast(`Conversation on - press ${keyLabel()} to talk`);
705
843
  }
706
844
 
@@ -184,15 +184,17 @@ export function createStabilityTracker() {
184
184
  // append nearly the whole transcript again whenever the re-decode
185
185
  // rewords the head and breaks every anchor - so replace instead.
186
186
  // Guarded by length so a collapsed model output can never wipe a good
187
- // transcript; tail-only and suspiciously short finals still go through
188
- // the normal join below.
187
+ // transcript. A full-window final shorter than the live assembly is a
188
+ // re-decode that dropped words: the preview the user watched was right,
189
+ // so keep it instead of submitting a shrunken transcript. Tail-only
190
+ // finals still go through the normal join below.
189
191
  if (
190
192
  windowHints &&
191
193
  windowHints.final === true &&
192
194
  (windowHints.absoluteStartMs ?? Infinity) <= 1000
193
195
  ) {
194
196
  const total = stableWords.length + tentativeWords.length;
195
- if (total === 0 || hypoWords.length * 2 >= total) {
197
+ if (total === 0 || hypoWords.length >= total) {
196
198
  try {
197
199
  sttLog(
198
200
  "transcript.final-replace",
@@ -205,10 +207,11 @@ export function createStabilityTracker() {
205
207
  }
206
208
  try {
207
209
  sttLog(
208
- "transcript.final-join",
210
+ "transcript.final-keep",
209
211
  `hypoWords=${hypoWords.length} assembledWords=${total} absoluteStartMs=${windowHints.absoluteStartMs}`,
210
212
  );
211
213
  } catch {}
214
+ return snapshot();
212
215
  }
213
216
 
214
217
  const assembled = stableWords.concat(tentativeWords);
package/lib/stt.js CHANGED
@@ -282,6 +282,9 @@ let pipelineGeneration = 0;
282
282
  // conversation loop instead of running the one-shot pipeline.
283
283
 
284
284
  let conversationHooks = { isActive: null, onKey: null };
285
+ let conversationStreamingEnabled = false;
286
+ let conversationStreamingStopSec = null;
287
+ let conversationStreamingOnStopped = null;
285
288
 
286
289
  export function __setConversationHooks(hooks) {
287
290
  conversationHooks = { ...conversationHooks, ...hooks };
@@ -550,6 +553,7 @@ let streamingToastBase = "";
550
553
  let streamingToastStartMs = null;
551
554
  let streamingLastPartialMs = null;
552
555
  let lastPartialText = null;
556
+ let lastAudioActiveMs = null;
553
557
  let streamingSlowAfterMs = 2000;
554
558
  const STREAMING_TOAST_DURATION = 3000;
555
559
  const STREAMING_TOAST_INTERVAL = 2500;
@@ -567,6 +571,7 @@ export function __clearStreamingToastState() {
567
571
  streamingToastBase = "";
568
572
  streamingToastStartMs = null;
569
573
  streamingLastPartialMs = null;
574
+ lastAudioActiveMs = null;
570
575
  lastPartialText = null;
571
576
  }
572
577
 
@@ -603,6 +608,7 @@ export function showStreamingToast(message, { slowAfterMs } = {}) {
603
608
  streamingToastBase = message;
604
609
  streamingToastStartMs = Date.now();
605
610
  streamingLastPartialMs = null;
611
+ lastAudioActiveMs = null;
606
612
  lastPartialText = null;
607
613
  if (Number(slowAfterMs) > 0) streamingSlowAfterMs = Math.floor(Number(slowAfterMs));
608
614
  if (!streamingToastFn) return;
@@ -641,6 +647,7 @@ export function clearStreamingToast() {
641
647
  streamingToastBase = "";
642
648
  streamingToastStartMs = null;
643
649
  streamingLastPartialMs = null;
650
+ lastAudioActiveMs = null;
644
651
  }
645
652
 
646
653
  export function formatElapsed(ms) {
@@ -1484,6 +1491,27 @@ async function transcribeApi(kv, logger) {
1484
1491
  return transcribeApiFile(wavFile, sttApiEndpoint, model, sttApiKeyEnv, logger);
1485
1492
  }
1486
1493
 
1494
+ /**
1495
+ * Which path owns silence auto-end for the current streaming session.
1496
+ * Conversation mode always wins: the plain dictation path submits the text
1497
+ * directly, which bypasses the conversation turn (no reply wait, no speech,
1498
+ * UI stuck on "Listening"). While a conversation is active the timer either
1499
+ * finishes the turn through the conversation callback or stays off - it never
1500
+ * falls back to that submit.
1501
+ */
1502
+ export function streamAutoEndRoute({
1503
+ conversation,
1504
+ conversationStopSec,
1505
+ conversationOnStopped,
1506
+ previewAutoEnd,
1507
+ }) {
1508
+ if (conversation) {
1509
+ const enabled = conversationStopSec > 0 || previewAutoEnd === true;
1510
+ return enabled && typeof conversationOnStopped === "function" ? "conversation" : "off";
1511
+ }
1512
+ return previewAutoEnd === true ? "dictation" : "off";
1513
+ }
1514
+
1487
1515
  export function insertIntoFocusedInput(renderer, text, submit = false) {
1488
1516
  const focused = renderer?.currentFocusedRenderable;
1489
1517
  if (!focused || typeof focused.insertText !== "function") return false;
@@ -1899,10 +1927,15 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
1899
1927
  disableStreamAutoEnd();
1900
1928
  return;
1901
1929
  }
1902
- // Only act if auto-end is actually enabled.
1903
- if (!preview?.getAutoEndOfSpeech?.()) return;
1904
- // Audio-level silence check: if the mic is picking up nothing,
1905
- // the user has stopped speaking regardless of what whisper returns.
1930
+ const route = streamAutoEndRoute({
1931
+ conversation: conversationActive(),
1932
+ conversationStopSec: conversationStreamingStopSec,
1933
+ conversationOnStopped: conversationStreamingOnStopped,
1934
+ previewAutoEnd: preview?.getAutoEndOfSpeech?.() === true,
1935
+ });
1936
+ if (route === "off") return;
1937
+ const useConv = route === "conversation";
1938
+ const convSec = conversationStreamingStopSec;
1906
1939
  let audioSilent = false;
1907
1940
  let rms = 0;
1908
1941
  try {
@@ -1918,19 +1951,49 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
1918
1951
  if (rms < 0.01) audioSilent = true;
1919
1952
  }
1920
1953
  } catch {}
1954
+ if (audioSilent) {
1955
+ if (lastAudioActiveMs === null) lastAudioActiveMs = Date.now();
1956
+ } else {
1957
+ lastAudioActiveMs = null;
1958
+ }
1959
+ const audioSilentMs = lastAudioActiveMs ? Date.now() - lastAudioActiveMs : 0;
1921
1960
  const last = streamingLastPartialMs;
1922
1961
  const elapsed = last ? Date.now() - last : 0;
1923
- if (audioSilent && last && elapsed > AUTO_END_SILENCE_MS) {
1962
+ const threshold = useConv && convSec > 0 ? convSec * 1000 : AUTO_END_SILENCE_MS;
1963
+ if (useConv) {
1964
+ if (audioSilent && audioSilentMs > threshold) {
1965
+ sttLog(
1966
+ "stream.autoEnd",
1967
+ `conversation triggered rms=${rms.toFixed(4)} silentMs=${audioSilentMs}ms`,
1968
+ );
1969
+ disableStreamAutoEnd();
1970
+ try {
1971
+ conversationStreamingOnStopped?.();
1972
+ } catch {}
1973
+ return;
1974
+ }
1975
+ if (last && elapsed > threshold * 2) {
1976
+ sttLog(
1977
+ "stream.autoEnd",
1978
+ `conversation triggered (partial) rms=${rms.toFixed(4)} elapsed=${elapsed}ms`,
1979
+ );
1980
+ disableStreamAutoEnd();
1981
+ try {
1982
+ conversationStreamingOnStopped?.();
1983
+ } catch {}
1984
+ }
1985
+ return;
1986
+ }
1987
+ if (audioSilent && audioSilentMs > threshold) {
1924
1988
  sttLog(
1925
1989
  "stream.autoEnd",
1926
- `triggered by silence (audio+partial) rms=${rms.toFixed(4)} elapsed=${elapsed}ms`,
1990
+ `triggered by silence (audio) rms=${rms.toFixed(4)} silentMs=${audioSilentMs}ms`,
1927
1991
  );
1928
1992
  disableStreamAutoEnd();
1929
1993
  void finalizeStreamingDictation({ submit: preview?.getAutoSubmit?.() ?? false });
1930
1994
  return;
1931
1995
  }
1932
- // Fallback: partial-based silence (no audio data available).
1933
- if (last && elapsed > AUTO_END_SILENCE_MS * 2) {
1996
+ if (last && elapsed > threshold * 2) {
1934
1997
  sttLog(
1935
1998
  "stream.autoEnd",
1936
1999
  `triggered by silence (partial only) rms=${rms.toFixed(4)} elapsed=${elapsed}ms`,
@@ -2007,11 +2070,10 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
2007
2070
  toast("Live notes recording - stop it first (/voice-notes-stop)", "warning");
2008
2071
  return false;
2009
2072
  }
2010
- if (conversationActive()) {
2073
+ if (conversationActive() && !conversationStreamingEnabled) {
2011
2074
  toast("Conversation mode is on - use the conversation key to exit", "warning");
2012
2075
  return false;
2013
2076
  }
2014
- streamStarting = true;
2015
2077
  const myGen = ++streamGeneration;
2016
2078
  // Model/language change tears down the warm lease first so the next
2017
2079
  // start loads the new model instead of reusing the stale server. The
@@ -2277,9 +2339,9 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
2277
2339
  // server lease is RETAINED across dictations (parked as the single warm
2278
2340
  // handle) and released only when the stop result proves the server itself
2279
2341
  // is at fault, on model/language change, or on plugin unload.
2280
- async function finalizeStreamingDictation({ submit = false } = {}) {
2342
+ async function finalizeStreamingDictation({ submit = false, insert = true } = {}) {
2281
2343
  if (!streamingDictationActive || !streamController) return null;
2282
- sttLog("stream.finalize", `submit=${submit}`);
2344
+ sttLog("stream.finalize", `submit=${submit} insert=${insert}`);
2283
2345
  disableStreamAutoEnd();
2284
2346
  const controller = streamController;
2285
2347
  const editor = streamEditor;
@@ -2360,7 +2422,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
2360
2422
  }
2361
2423
  if (fin?.inserted) {
2362
2424
  logger?.log("STT", `Streaming finalized via owned range chars=${text.length}`, "debug");
2363
- if (submit || autoSubmitNow()) {
2425
+ if (submit === true || (submit !== false && autoSubmitNow())) {
2364
2426
  if (target && typeof target.submit === "function") {
2365
2427
  try {
2366
2428
  target.submit();
@@ -2382,41 +2444,45 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
2382
2444
  // missing/unusable target retains the transcript via toast instead of
2383
2445
  // redirecting into chat or the current focus.
2384
2446
  let inserted = false;
2385
- if (target && typeof target.insertText === "function") {
2386
- try {
2387
- target.insertText(text);
2388
- inserted = true;
2389
- } catch {
2390
- inserted = false;
2447
+ if (insert) {
2448
+ if (target && typeof target.insertText === "function") {
2449
+ try {
2450
+ target.insertText(text);
2451
+ inserted = true;
2452
+ } catch {
2453
+ inserted = false;
2454
+ }
2455
+ } else {
2456
+ inserted = insertIntoFocusedInput(renderer, text, false);
2391
2457
  }
2392
- } else {
2393
- inserted = insertIntoFocusedInput(renderer, text, false);
2394
2458
  }
2395
- if (!inserted) {
2459
+ if (insert && !inserted) {
2396
2460
  logger?.log("STT", "Streaming finalize: no editable field, transcript retained", "warn");
2397
2461
  toast("No editable field - dictated text kept (not inserted elsewhere)", "warning");
2398
2462
  if (preview) processing = false;
2399
2463
  retainWarmStreamController(controller);
2400
2464
  return { text, inserted: false };
2401
2465
  }
2402
- logger?.log("STT", `Streaming finalized chars=${text.length}`, "debug");
2403
- if (submit || autoSubmitNow()) {
2404
- if (target && typeof target.submit === "function") {
2405
- try {
2406
- target.submit();
2407
- toast("Streaming transcription submitted", "success");
2408
- } catch (err) {
2409
- toast(`Streaming submit failed: ${err?.message || err}`, "error");
2466
+ if (insert) {
2467
+ logger?.log("STT", `Streaming finalized chars=${text.length}`, "debug");
2468
+ if (submit === true || (submit !== false && autoSubmitNow())) {
2469
+ if (target && typeof target.submit === "function") {
2470
+ try {
2471
+ target.submit();
2472
+ toast("Streaming transcription submitted", "success");
2473
+ } catch (err) {
2474
+ toast(`Streaming submit failed: ${err?.message || err}`, "error");
2475
+ }
2476
+ } else {
2477
+ toast("Target cannot submit - text inserted, not submitted", "warning");
2410
2478
  }
2411
2479
  } else {
2412
- toast("Target cannot submit - text inserted, not submitted", "warning");
2480
+ toast("Streaming transcription added", "success");
2413
2481
  }
2414
- } else {
2415
- toast("Streaming transcription added", "success");
2416
2482
  }
2417
2483
  if (preview) processing = false;
2418
2484
  retainWarmStreamController(controller);
2419
- return { text, inserted: true };
2485
+ return { text, inserted };
2420
2486
  }
2421
2487
 
2422
2488
  // Cancel: drop the session and remove ONLY plugin-owned dictated text.
@@ -2501,11 +2567,22 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
2501
2567
  isRecording: () => recording,
2502
2568
  isProcessing: () => processing,
2503
2569
  isStreaming,
2570
+ log: (msg) => sttLog("CONV", msg),
2504
2571
  start: startLoopRecording,
2505
2572
  setSilenceAutoFinalize: (sec, onStopped) => {
2506
2573
  conversationSilenceStopSec = sec && sec > 0 ? sec : null;
2507
2574
  conversationOnStopped = typeof onStopped === "function" ? onStopped : null;
2508
2575
  },
2576
+ setConversationStreamingFinalize: (sec, onStopped) => {
2577
+ conversationStreamingStopSec = sec && sec > 0 ? sec : null;
2578
+ conversationStreamingOnStopped = typeof onStopped === "function" ? onStopped : null;
2579
+ if (conversationStreamingStopSec) enableStreamAutoEnd();
2580
+ else disableStreamAutoEnd();
2581
+ },
2582
+ setConversationStreamingEnabled: (v) => {
2583
+ conversationStreamingEnabled = !!v;
2584
+ },
2585
+ finalizeStreaming: (opts) => finalizeStreamingDictation(opts || {}),
2509
2586
  setStopHint: (hint) => __setStopHint(hint),
2510
2587
  cancel: () => cancelRecording(logger),
2511
2588
  discard: () => discardTurn(),
package/lib/tts-player.js CHANGED
@@ -80,7 +80,9 @@ export function measureWavDurationSec(wav) {
80
80
  // header at load), so sox needs no `-t raw` format declaration and the
81
81
  // argv here is the complete command.
82
82
  export function buildPlayArgs({ offsetSec = 0, rate = 1 } = {}) {
83
- const argv = ["-q", "-"];
83
+ // Small read-ahead buffer so SIGSTOP lands closer to the audible cursor;
84
+ // the default (8192) lets ~0.4s of audio queue ahead of the process.
85
+ const argv = ["-q", "--buffer", "1024", "-"];
84
86
  const offset = Number(offsetSec) || 0;
85
87
  if (offset > 0) argv.push("trim", String(offset));
86
88
  const speed = Number(rate) || 1;
package/lib/tts.js CHANGED
@@ -393,40 +393,85 @@ export async function getTurnAssistantText(client, api) {
393
393
  if (route.name !== "session") return null;
394
394
 
395
395
  const sessionID = route.params.sessionID;
396
- const stateMessages = api.state.session.messages(sessionID);
397
- if (!stateMessages || stateMessages.length === 0) return null;
398
-
399
- const assistantIDs = [];
400
- for (let i = stateMessages.length - 1; i >= 0; i--) {
401
- if (stateMessages[i].role === "user") break;
402
- if (stateMessages[i].role === "assistant") {
403
- assistantIDs.unshift(stateMessages[i].id);
396
+
397
+ // The message cache can trail the execution-completed signal by a beat, so a
398
+ // single read here can still show the previous turn and report "no reply".
399
+ // Retry briefly instead of dropping the answer.
400
+ let stateMessages = [];
401
+ let assistantIDs = [];
402
+ for (let attempt = 0; attempt < 4; attempt += 1) {
403
+ stateMessages = api.state.session.messages(sessionID) || [];
404
+ assistantIDs = [];
405
+ for (let i = stateMessages.length - 1; i >= 0; i -= 1) {
406
+ if (stateMessages[i].role === "user") break;
407
+ if (stateMessages[i].role === "assistant") {
408
+ assistantIDs.unshift(stateMessages[i].id);
409
+ }
404
410
  }
411
+ if (assistantIDs.length > 0) break;
412
+ if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, 150));
405
413
  }
414
+ sttLog(
415
+ "tts.conv",
416
+ `getTurnAssistantText session=${sessionID} n=${stateMessages.length} assistants=${assistantIDs.length}`,
417
+ );
406
418
  if (assistantIDs.length === 0) return null;
407
419
 
408
- const allText = [];
409
- for (const msgID of assistantIDs) {
410
- try {
411
- const fullMsg = await client.session
412
- .message({ sessionID, messageID: msgID }, { throwOnError: true })
413
- .then((r) => r.data);
420
+ // The answer is the last assistant message that finished WITHOUT asking for
421
+ // more tool calls. Earlier ones are mid-turn commentary ("let me check...")
422
+ // and reading them aloud reads half a thought. Fall back to the whole turn
423
+ // when no message carries a terminal finish, or when the terminal one holds
424
+ // no prose (a tool-only finish).
425
+ const terminalIDs = assistantIDs.filter((id) => {
426
+ const finish = stateMessages.find((m) => m?.id === id)?.finish;
427
+ return !!finish && finish !== "tool-calls" && finish !== "tool_calls";
428
+ });
429
+ const answerID = terminalIDs.length > 0 ? terminalIDs[terminalIDs.length - 1] : null;
414
430
 
431
+ const textOf = async (msgID) => {
432
+ let fullMsg = stateMessages.find((m) => m?.id === msgID);
433
+ if (!fullMsg || !(fullMsg.parts && fullMsg.parts.length)) {
434
+ try {
435
+ fullMsg = await client.session
436
+ .message({ sessionID, messageID: msgID }, { throwOnError: true })
437
+ .then((r) => r.data);
438
+ } catch {
439
+ fullMsg = null;
440
+ }
441
+ }
442
+ try {
415
443
  const textParts = (fullMsg?.parts || []).filter((p) => p.type === "text");
416
- const text = textParts
444
+ sttLog(
445
+ "tts.conv",
446
+ `msg=${msgID} parts=${fullMsg?.parts?.length ?? 0} textParts=${textParts.length}`,
447
+ );
448
+ return textParts
417
449
  .map((p) => p.text || "")
418
450
  .join("\n\n")
419
451
  .trim();
420
- if (text) allText.push(text);
421
452
  } catch {
422
- // Skip messages that fail to fetch
453
+ return "";
454
+ }
455
+ };
456
+
457
+ let spokenIDs = answerID ? [answerID] : assistantIDs;
458
+ let allText = [];
459
+ for (const msgID of spokenIDs) {
460
+ const text = await textOf(msgID);
461
+ if (text) allText.push(text);
462
+ }
463
+ if (allText.length === 0 && answerID) {
464
+ spokenIDs = assistantIDs;
465
+ for (const msgID of spokenIDs) {
466
+ const text = await textOf(msgID);
467
+ if (text) allText.push(text);
423
468
  }
424
469
  }
425
470
 
426
471
  if (allText.length === 0) return null;
427
472
 
428
473
  return {
429
- lastMessageID: assistantIDs[assistantIDs.length - 1],
474
+ lastMessageID: spokenIDs[spokenIDs.length - 1],
430
475
  text: allText.join("\n\n"),
431
476
  };
432
477
  }
@@ -799,9 +844,13 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
799
844
 
800
845
  function getChatterboxClient() {
801
846
  if (!chatterboxClient) {
847
+ const defaultChatterboxPython =
848
+ "/Users/duytrinh/.local/share/opencode-voice/chatterbox-venv/bin/python";
802
849
  chatterboxClient = createChatterbox({
803
850
  variant: opts?.ttsChatterboxVariant,
804
- python: opts?.ttsChatterboxPython,
851
+ python:
852
+ opts?.ttsChatterboxPython ||
853
+ (fs.existsSync(defaultChatterboxPython) ? defaultChatterboxPython : undefined),
805
854
  voiceRef: opts?.ttsChatterboxVoiceRef,
806
855
  logger,
807
856
  });
@@ -866,9 +915,13 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
866
915
 
867
916
  function getVieneuClient() {
868
917
  if (!vieneuClient) {
918
+ const defaultVieneuPython =
919
+ "/Users/duytrinh/.local/share/opencode-voice/vieneu-venv/bin/python";
869
920
  vieneuClient = createVieneu({
870
- python: opts?.ttsVieneuPython,
871
- voice: opts?.ttsVieneuVoice,
921
+ python:
922
+ opts?.ttsVieneuPython ||
923
+ (fs.existsSync(defaultVieneuPython) ? defaultVieneuPython : undefined),
924
+ voice: kv.get("tts.vieneuVoice", opts?.ttsVieneuVoice || "Mai Anh"),
872
925
  logger,
873
926
  });
874
927
  }
@@ -1278,8 +1331,10 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
1278
1331
  }
1279
1332
 
1280
1333
  async function speakAssistantTurn() {
1334
+ sttLog("tts.conv", "speakAssistantTurn start");
1281
1335
  const tFetch = Date.now();
1282
1336
  const result = await getTurnAssistantText(client, api);
1337
+ sttLog("tts.conv", `speakAssistantTurn result=${!!result} text=${result?.text?.length ?? 0}`);
1283
1338
  if (!result || !result.text) {
1284
1339
  logger?.log?.("TTS", "Conversation: no assistant text to speak", "warn");
1285
1340
  return { spoken: false };
@@ -1298,6 +1353,7 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
1298
1353
  return { spoken: false };
1299
1354
  }
1300
1355
  if (!announceNormalize("Conversation", llmResult)) {
1356
+ sttLog("tts.conv", "announceNormalize failed");
1301
1357
  clearProcessingToast();
1302
1358
  return { spoken: false };
1303
1359
  }
@@ -1428,6 +1484,34 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
1428
1484
 
1429
1485
  // ---- Commands ----
1430
1486
 
1487
+ const VIENEU_VOICES = [
1488
+ "Adam bựa",
1489
+ "Trúc Ly",
1490
+ "Thiện Minh",
1491
+ "Mai Anh",
1492
+ "Hải Đăng",
1493
+ "Thùy Dung",
1494
+ "Thiền Tâm Đức",
1495
+ "Ngọc Huyền",
1496
+ "Quang Sơn",
1497
+ "Ngọc Trân",
1498
+ "Minh Đức",
1499
+ "Phạm Tuyên",
1500
+ "Thái Sơn",
1501
+ "Xuân Vĩnh",
1502
+ "Thanh Bình",
1503
+ "Ngọc Linh",
1504
+ "Đoan Trang",
1505
+ "Thục Đoan",
1506
+ "Minh Triết",
1507
+ "Mỹ Duyên",
1508
+ "Quỳnh Anh",
1509
+ "Đức Trí",
1510
+ "Kim Thanh",
1511
+ "Adam",
1512
+ "Quốc Tuấn",
1513
+ ];
1514
+
1431
1515
  const commands = [
1432
1516
  {
1433
1517
  title: "TTS: speak last response",
@@ -1614,6 +1698,33 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
1614
1698
  );
1615
1699
  },
1616
1700
  },
1701
+ {
1702
+ title: "TTS: select VieNeu voice",
1703
+ value: "tts.vieneu-voice",
1704
+ category: "opencode-voice",
1705
+ description: "Choose VieNeu-TTS voice (bilingual vi+en) - persisted",
1706
+ slash: { name: "tts-vieneu-voice" },
1707
+ onSelect() {
1708
+ const current = kv.get("tts.vieneuVoice", opts?.ttsVieneuVoice || "Mai Anh");
1709
+ api.ui.dialog.replace(() =>
1710
+ api.ui.DialogSelect({
1711
+ title: "Select VieNeu voice",
1712
+ current,
1713
+ options: VIENEU_VOICES.map((name) => ({
1714
+ title: name,
1715
+ value: name,
1716
+ onSelect() {
1717
+ kv.set("tts.vieneuVoice", name);
1718
+ // Reset client so next speak uses new voice.
1719
+ vieneuClient = null;
1720
+ toast(`VieNeu voice: ${name}`);
1721
+ api.ui.dialog.clear();
1722
+ },
1723
+ })),
1724
+ }),
1725
+ );
1726
+ },
1727
+ },
1617
1728
  ];
1618
1729
 
1619
1730
  return { commands, controller };
@@ -20,7 +20,7 @@ const DEFAULT_PORT = 8000;
20
20
  const DEFAULT_PORT_SCAN_MAX = 4;
21
21
  const READY_TIMEOUT_MS = 300000;
22
22
  const READY_POLL_MS = 1000;
23
- const SYNTHESIZE_TIMEOUT_MS = 30000;
23
+ const SYNTHESIZE_TIMEOUT_MS = 120000;
24
24
 
25
25
  // VieNeu-TTS supports Vietnamese + English code-switching natively.
26
26
  export const VIENEU_LANGUAGES = new Set(["vi", "en"]);
@@ -65,14 +65,15 @@ export function createVieneuClient(options = {}) {
65
65
 
66
66
  // Default to the venv python if it exists (host truncates plugin options,
67
67
  // so ttsVieneuPython may not be delivered).
68
- const defaultPython = "/Users/duytrinh/.local/share/opencode-voice/vieneu-venv/bin/python";
69
- const resolvedPython = python || (fs.existsSync(defaultPython) ? defaultPython : "python3");
70
68
 
71
69
  const spawnFn = deps.spawn ?? spawn;
72
70
  const fetchFn = deps.fetch ?? fetch;
73
71
 
74
72
  let proc = null;
75
73
  let ready = false;
74
+ // True when `ready` points at a sidecar this client did not spawn (it was
75
+ // already serving the port). Such a server is never killed by stop().
76
+ let adopted = false;
76
77
  let ownerPid = null;
77
78
  let boundPort = options.port ?? DEFAULT_PORT;
78
79
  let lastError = null;
@@ -116,6 +117,16 @@ export function createVieneuClient(options = {}) {
116
117
  async function startOnPort(port) {
117
118
  boundPort = port;
118
119
  if (await portResponds(port)) {
120
+ // A healthy sidecar already serves this port (earlier plugin session,
121
+ // manual run, another opencode instance). Adopt it. The app binds its
122
+ // own default port regardless of flags, so a second child cannot start
123
+ // beside it: it exits and a working sidecar becomes a Piper fallback.
124
+ if ((await probeOnce()) === "ready") {
125
+ adopted = true;
126
+ ready = true;
127
+ ownerPid = null;
128
+ return { ok: true, adopted: true };
129
+ }
119
130
  return {
120
131
  ok: false,
121
132
  code: "PORT_IN_USE",
@@ -123,7 +134,8 @@ export function createVieneuClient(options = {}) {
123
134
  message: `Port ${port} already answers`,
124
135
  };
125
136
  }
126
- const bin = resolvedPython;
137
+ adopted = false;
138
+ const bin = python || "python3";
127
139
  try {
128
140
  proc = spawnFn(bin, ["-m", "apps.openai_speech"], {
129
141
  stdio: ["ignore", "ignore", "pipe"],
@@ -184,7 +196,7 @@ export function createVieneuClient(options = {}) {
184
196
  }
185
197
 
186
198
  async function start() {
187
- if (proc && ready) return true;
199
+ if (ready) return true;
188
200
  const scanMax = Math.max(1, Math.floor(portScanMax) || 1);
189
201
  for (let i = 0; i < scanMax; i++) {
190
202
  const port = boundPort + i;
@@ -249,6 +261,11 @@ export function createVieneuClient(options = {}) {
249
261
  return { ok: true, wav: audio };
250
262
  } catch (err) {
251
263
  const timedOut = err?.name === "TimeoutError" || err?.name === "AbortError";
264
+ if (adopted && !timedOut) {
265
+ // The sidecar we adopted is gone; let the next turn start one.
266
+ ready = false;
267
+ adopted = false;
268
+ }
252
269
  return {
253
270
  ok: false,
254
271
  code: timedOut ? "SYNTH_TIMEOUT" : "REQUEST_FAILED",
@@ -259,6 +276,7 @@ export function createVieneuClient(options = {}) {
259
276
 
260
277
  function stop() {
261
278
  ready = false;
279
+ adopted = false;
262
280
  ownerPid = null;
263
281
  if (proc) {
264
282
  const owned = proc;
@@ -270,7 +288,7 @@ export function createVieneuClient(options = {}) {
270
288
  }
271
289
 
272
290
  function isRunning() {
273
- return ready && proc !== null;
291
+ return ready && (proc !== null || adopted);
274
292
  }
275
293
 
276
294
  function getLastError() {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bojackduy/opencode-voice",
3
- "version": "0.11.4",
3
+ "version": "0.11.5",
4
4
  "description": "Speech-to-text and text-to-speech for OpenCode. Record voice prompts with whisper-cpp, hear responses via Piper TTS, with LLM normalization through any OpenAI-compatible endpoint.",
5
5
  "keywords": [
6
6
  "accessibility",
@@ -57,10 +57,13 @@
57
57
  "prepack": "npm run check && npm test",
58
58
  "release:patch": "npm run check && npm test && npm version patch -m \"chore: release %s\" && git push fork master && git push fork $(git describe --tags --abbrev=0)",
59
59
  "release:minor": "npm run check && npm test && npm version minor -m \"chore: release %s\" && git push fork master && git push fork $(git describe --tags --abbrev=0)",
60
- "release:major": "npm run check && npm test && npm version major -m \"chore: release %s\" && git push fork master && git push fork $(git describe --tags --abbrev=0)"
60
+ "release:major": "npm run check && npm test && npm version major -m \"chore: release %s\" && git push fork master && git push fork $(git describe --tags --abbrev=0)",
61
+ "deploy": "wrangler deploy",
62
+ "preview": "wrangler dev"
61
63
  },
62
64
  "devDependencies": {
63
65
  "oxfmt": "0.50.0",
64
- "oxlint": "1.65.0"
66
+ "oxlint": "1.65.0",
67
+ "wrangler": "^4.156.0"
65
68
  }
66
69
  }