@bojackduy/opencode-voice 0.11.3 → 0.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,13 +23,11 @@
23
23
  // or the global /voice-cancel. Empty or failed turns pause instead of
24
24
  // auto-recording, so the key never surprises.
25
25
  //
26
- // Latency: while waiting, assistant text deltas are spoken sentence by
27
- // sentence (local cleanup, no LLM), so the answer starts before the turn
28
- // completes. Replies with nothing streamable fall back to the full
29
- // LLM-narrated speak.
26
+ // Latency vs. honesty: the turn is voiced only after the whole execution
27
+ // (tool calls included) completes, so what is read aloud is the answer and
28
+ // never mid-turn commentary.
30
29
 
31
30
  import { clearProcessingToast, showProcessingToast } from "./stt.js";
32
- import { isSpeakableSentence, localSpeechCleanup, splitSpokenSentences } from "./tts.js";
33
31
 
34
32
  export const DEFAULT_STOP_PHRASES = [
35
33
  "stop",
@@ -105,28 +103,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
105
103
  let active = false;
106
104
  let phase = "idle"; // idle | recording | processing | waiting | speaking | paused
107
105
 
108
- // Reply streaming state: text deltas heard while the agent works are spoken
109
- // sentence by sentence through a serial queue, so the answer starts before
110
- // the turn completes and every sentence gets its own vi/en voice.
111
-
112
- let streamBuffer = "";
113
- let streamSpokenChars = 0;
106
+ // Event subscriptions owned by the current wait/probe set. Nothing is
107
+ // spoken from live deltas: the agent's commentary during tool calls is not
108
+ // the answer, so only the completed turn is voiced (see speakReply).
114
109
  let streamUnsubs = [];
115
- let speakTail = Promise.resolve();
116
110
  let speakGen = 0;
117
111
 
118
- function queueSpeak(sentence) {
119
- const g = speakGen;
120
- speakTail = speakTail.then(() => {
121
- // Speaking must also drain: finishTurnFlow flips to speaking before
122
- // flushing the tail, so a waiting-only guard would skip queued
123
- // sentences/final partials under real TTS latency. speakGen still
124
- // invalidates on pause/stop.
125
- if (g !== speakGen || !active || (phase !== "waiting" && phase !== "speaking")) return null;
126
- return tts.speak(sentence);
127
- });
128
- return speakTail;
129
- }
130
112
  let turn = 0;
131
113
  let consecErrors = 0;
132
114
  let generation = 0;
@@ -136,6 +118,44 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
136
118
  api.ui.toast({ message, variant, duration: 3000 });
137
119
  }
138
120
 
121
+ // v2 persistent status line: phase-aware buttons. v1 has no surface, so
122
+ // these are no-ops there (api.conversationStatus is undefined).
123
+ const convStatus = api.conversationStatus || null;
124
+ const clog = (m) => {
125
+ try {
126
+ stt.log?.(m);
127
+ } catch {}
128
+ };
129
+ const setConvPhase = api.setConvPhase || (() => {});
130
+ const setConvActions = api.setConvActions || (() => {});
131
+ function convBegin() {
132
+ try {
133
+ const p = convStatus?.begin?.();
134
+ if (p && typeof p.then === "function") p.catch(() => {});
135
+ } catch {}
136
+ }
137
+ function convUpdate() {
138
+ try {
139
+ convStatus?.update?.();
140
+ } catch {}
141
+ }
142
+ function convClear() {
143
+ try {
144
+ convStatus?.clear?.();
145
+ } catch {}
146
+ }
147
+ function convSetPhase(p) {
148
+ try {
149
+ setConvPhase(p);
150
+ convUpdate();
151
+ } catch {}
152
+ }
153
+ function convSetActions(actions) {
154
+ try {
155
+ setConvActions(actions);
156
+ } catch {}
157
+ }
158
+
139
159
  function currentSessionID() {
140
160
  const route = api?.route?.current;
141
161
  return route?.name === "session" ? route?.params?.sessionID : null;
@@ -155,14 +175,87 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
155
175
  if (stt.isRecording() || stt.isProcessing() || stt.isStreaming?.()) return;
156
176
  turn += 1;
157
177
  phase = "recording";
158
- logger?.log("VOICE", `Conversation turn ${turn} listening`, "debug");
159
- if (!stt.start()) {
160
- phase = "idle";
161
- stop("busy");
162
- toast("STT busy, conversation off", "warning");
163
- return;
178
+ clog(`Conversation turn ${turn} listening`);
179
+ const streamingMode = (api?.kv?.get?.("stt.mode") ?? opts?.sttMode) === "streaming";
180
+ // Allow streaming STT during conversation for waveform + live transcript.
181
+ try {
182
+ stt.setConversationStreamingEnabled?.(true);
183
+ } catch {}
184
+ const autoEnd = api?.kv?.get?.("stt.autoEndOfSpeech") === true;
185
+ if (streamingMode) {
186
+ // Register the stop callback even when auto-end is off: the preview
187
+ // toggle can turn auto-end on mid-recording, and the STT timer must
188
+ // then finish this turn through the conversation loop instead of
189
+ // submitting the text as plain dictation (which skips the reply wait,
190
+ // the speech, and leaves the UI on "Listening").
191
+ const onStreamAutoStop = () => {
192
+ clog(`Conversation streaming auto-finalize callback active=${active} phase=${phase}`);
193
+ if (active && phase === "recording") {
194
+ clog("Conversation auto-finalize on silence (streaming)");
195
+ void finishTurnFlow();
196
+ }
197
+ };
198
+ stt.setConversationStreamingFinalize?.(autoEnd ? 2.5 : null, onStreamAutoStop);
199
+ const started = stt.startStreaming?.();
200
+ if (!started) {
201
+ // Fallback to batch recording.
202
+ stt.setConversationStreamingFinalize?.(null, null);
203
+ if (autoEnd) {
204
+ stt.setSilenceAutoFinalize?.(2.5, () => {
205
+ if (active && phase === "recording") {
206
+ clog("Conversation auto-finalize on silence");
207
+ finishTurnFlow();
208
+ }
209
+ });
210
+ } else {
211
+ stt.setSilenceAutoFinalize?.(null, null);
212
+ }
213
+ if (!stt.start()) {
214
+ phase = "idle";
215
+ stop("busy");
216
+ toast("STT busy, conversation off", "warning");
217
+ return;
218
+ }
219
+ }
220
+ } else {
221
+ if (autoEnd) {
222
+ stt.setSilenceAutoFinalize?.(2.5, () => {
223
+ if (active && phase === "recording") {
224
+ clog("Conversation auto-finalize on silence");
225
+ finishTurnFlow();
226
+ }
227
+ });
228
+ } else {
229
+ stt.setSilenceAutoFinalize?.(null, null);
230
+ }
231
+ if (!stt.start()) {
232
+ phase = "idle";
233
+ stop("busy");
234
+ toast("STT busy, conversation off", "warning");
235
+ return;
236
+ }
164
237
  }
165
- toast(`Listening - press ${keyLabel()} to send`);
238
+ convSetActions({
239
+ cancel: () => {
240
+ if (phase === "recording") {
241
+ try {
242
+ stt.cancelStreaming?.();
243
+ } catch {}
244
+ stt.cancel();
245
+ phase = "paused";
246
+ convSetPhase("paused");
247
+ }
248
+ },
249
+ send: () => {
250
+ if (phase === "recording") finishTurnFlow();
251
+ },
252
+ });
253
+ convSetPhase("recording");
254
+ toast(
255
+ autoEnd
256
+ ? `Listening - speak, then pause 2.5s to send (or press ${keyLabel()})`
257
+ : `Listening - press ${keyLabel()} to send`,
258
+ );
166
259
  }
167
260
 
168
261
  // The resting state: mic closed, one press to talk. Entered by start() and
@@ -170,6 +263,13 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
170
263
 
171
264
  function enterPaused() {
172
265
  phase = "paused";
266
+ convSetActions({
267
+ talk: () => {
268
+ if (phase === "paused") beginTurn();
269
+ },
270
+ end: () => stop("user"),
271
+ });
272
+ convSetPhase("paused");
173
273
  toast(`Press ${keyLabel()} to talk`);
174
274
  }
175
275
 
@@ -178,7 +278,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
178
278
  // means "I want to talk now", so it must never exit silently.
179
279
 
180
280
  function bargeIn() {
181
- logger?.log("VOICE", "Conversation barge-in", "debug");
281
+ clog("Conversation barge-in");
282
+ stt.setSilenceAutoFinalize?.(null, null);
283
+ stt.setConversationStreamingFinalize?.(null, null);
284
+ try {
285
+ stt.cancelStreaming?.();
286
+ } catch {}
182
287
  discardStreamAudio();
183
288
  clearProcessingToast();
184
289
  // Settle the pending reply wait as "stopped" so the abandoned answer is
@@ -214,6 +319,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
214
319
  active = false;
215
320
  generation += 1;
216
321
  phase = "idle";
322
+ stt.setSilenceAutoFinalize?.(null, null);
323
+ stt.setConversationStreamingFinalize?.(null, null);
324
+ try {
325
+ stt.setConversationStreamingEnabled?.(false);
326
+ } catch {}
217
327
  if (waitCancel) {
218
328
  waitCancel("stopped");
219
329
  waitCancel = null;
@@ -221,71 +331,66 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
221
331
  endReplyStream();
222
332
  speakGen += 1;
223
333
  stt.setStopHint(null);
334
+ try {
335
+ stt.cancelStreaming?.();
336
+ } catch {}
224
337
  if (stt.isRecording() || stt.isProcessing()) stt.cancel();
225
338
  clearProcessingToast();
226
339
  tts.stop();
227
340
  tts.setConversationActive(false);
228
- logger?.log("VOICE", `Conversation stopped reason=${reason}`, "debug");
229
- }
230
-
231
- function startReplyStream(sessionID, turnStart) {
232
- streamBuffer = "";
233
- streamSpokenChars = 0;
234
- const unsub = api.event.on("message.part.delta", (event) => {
235
- try {
236
- if (!active || phase !== "waiting") return;
237
- const props = event.properties || {};
238
- const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
239
- if (typeof sessionID === "function" && !expectedID) return;
240
- if (expectedID && props.sessionID && props.sessionID !== expectedID) return;
241
- if (typeof props.delta !== "string" || !props.delta) return;
242
- if (props.field && props.field !== "text") return;
243
- const lookupID = props.sessionID || expectedID;
244
- const messages = api?.state?.session?.messages?.(lookupID) || [];
245
- const msg = messages.find((m) => m?.id === props.messageID);
246
- if (!msg || msg.role !== "assistant") return;
247
- if ((msg.time?.created || 0) < turnStart) return;
248
- const parts = api?.state?.part?.(props.messageID) || [];
249
- const part = parts.find((p) => p?.id === props.partID);
250
- if (part && part.type !== "text") return;
251
- streamBuffer += props.delta;
252
- flushStream(false);
253
- } catch {}
254
- });
255
- streamUnsubs.push(unsub);
256
- }
257
-
258
- function flushStream(final = false) {
259
- const { sentences, rest } = splitSpokenSentences(streamBuffer);
260
- const ready = final && rest.trim() ? [...sentences, rest] : sentences;
261
- if (ready.length > 0 && streamSpokenChars === 0) {
262
- showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
263
- }
264
- for (const sentence of ready) {
265
- if (!isSpeakableSentence(sentence)) continue;
266
- const cleaned = localSpeechCleanup(sentence);
267
- if (!cleaned) continue;
268
- queueSpeak(cleaned);
269
- streamSpokenChars += cleaned.length;
270
- }
271
- streamBuffer = final ? "" : rest;
341
+ convClear();
342
+ clog(`Conversation stopped reason=${reason}`);
272
343
  }
273
344
 
274
345
  // Wait for the submitted turn to finish: idle, a permission/question gate,
275
346
  // or timeout. Resolves early when stop() cancels the wait. gates:false drops
276
347
  // the permission/question subscriptions, so a second wait survives the gate
277
348
  // the user is answering on screen and only ends on idle/timeout/stop.
278
-
279
- function waitForReply(sessionID, { gates = true } = {}) {
349
+ //
350
+ // Completion is a host fact, not a guess: v2 emits
351
+ // `session.execution.succeeded|failed|interrupted` only when the whole
352
+ // execution - tool calls included - ends. The message list is the fallback
353
+ // signal, and BOTH paths are gated on `baseline` (the message count taken
354
+ // before submit), so a finished assistant message from an earlier turn can
355
+ // never be mistaken for this turn's reply. That gate is the fix for the
356
+ // reply being read ~800ms early with zero assistant messages in it.
357
+
358
+ function waitForReply(sessionID, { gates = true, baseline = 0 } = {}) {
280
359
  let settled = false;
281
360
  let unsubs = [];
282
361
  let timer = null;
362
+ let pollTimer = null;
363
+ let settleTimer = null;
364
+ let currentExecutionID = null;
365
+ let sawExecutionStart = false;
366
+
367
+ const resolveSessionID = () => (typeof sessionID === "function" ? sessionID() : sessionID);
368
+ const baselineFor = (sid) => (typeof baseline === "function" ? baseline(sid) : baseline);
369
+
370
+ // Assistant messages that arrived AFTER submit, oldest first.
371
+ const newAssistants = () => {
372
+ try {
373
+ const sid = resolveSessionID();
374
+ const msgs = api?.state?.session?.messages?.(sid) || [];
375
+ return msgs.slice(baselineFor(sid)).filter((m) => m?.role === "assistant");
376
+ } catch {
377
+ return [];
378
+ }
379
+ };
380
+ const isTerminal = (finish) => !!finish && finish !== "tool-calls" && finish !== "tool_calls";
381
+ const lastFinish = () => {
382
+ const list = newAssistants();
383
+ return list.length ? list[list.length - 1]?.finish : null;
384
+ };
385
+ const turnFinished = () => isTerminal(lastFinish());
283
386
 
284
387
  let cancelFn = null;
285
388
  const done = (outcome) => {
286
389
  if (settled) return null;
287
390
  settled = true;
288
391
  if (timer) clearTimeout(timer);
392
+ if (settleTimer) clearInterval(settleTimer);
393
+ if (pollTimer) clearInterval(pollTimer);
289
394
  for (const unsub of unsubs) {
290
395
  try {
291
396
  unsub();
@@ -304,20 +409,93 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
304
409
  cancelFn = (outcome) => finish(outcome || "stopped");
305
410
  waitCancel = cancelFn;
306
411
 
412
+ // The host event can land a beat before the message cache does. Give the
413
+ // message at most 2s to appear, then settle anyway: the execution is
414
+ // over, and "no reply to speak" is a truthful outcome.
415
+ const settle = (outcome) => {
416
+ if (settled) return;
417
+ if (turnFinished()) {
418
+ finish(outcome);
419
+ return;
420
+ }
421
+ if (settleTimer) return;
422
+ const startedAt = Date.now();
423
+ settleTimer = setInterval(() => {
424
+ if (settled) return;
425
+ if (turnFinished() || Date.now() - startedAt > 2000) {
426
+ clog(`Conversation settled outcome=${outcome} finish=${lastFinish() ?? "none"}`);
427
+ finish(outcome);
428
+ }
429
+ }, 100);
430
+ if (typeof settleTimer.unref === "function") settleTimer.unref();
431
+ };
432
+
307
433
  const forSession = (props) => {
308
- const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
309
- if (typeof sessionID === "function" && !expectedID) return false;
310
- return !expectedID || !props?.sessionID || props.sessionID === expectedID;
434
+ const sid = props?.sessionID ?? props?.id ?? props?.session?.id;
435
+ const expectedID = resolveSessionID();
436
+ if (typeof sessionID === "function" && !expectedID) return true;
437
+ if (!expectedID) return true;
438
+ if (!sid) return true;
439
+ return sid === expectedID;
440
+ };
441
+
442
+ const executionIDOf = (props) =>
443
+ props?.executionID ?? props?.executionId ?? props?.id ?? props?.execution?.id;
444
+
445
+ const onExecutionEnd = (outcome) => (event) => {
446
+ const props = event?.properties || event?.data || {};
447
+ if (!forSession(props)) return;
448
+ const eid = executionIDOf(props);
449
+ if (currentExecutionID && eid && eid !== currentExecutionID) {
450
+ clog(
451
+ `Conversation execution ${outcome} ignored stale id=${eid} current=${currentExecutionID}`,
452
+ );
453
+ return;
454
+ }
455
+ // The host's execution events carry no execution id, so the guard is
456
+ // structural: a terminal event is only trusted once this wait has seen
457
+ // execution.started, or once this turn's own assistant message is
458
+ // there with a terminal finish. Otherwise it is a late event from the
459
+ // turn that barge-in interrupted.
460
+ if (!sawExecutionStart && !turnFinished()) {
461
+ clog(`Conversation execution ${outcome} ignored (no start, no new assistant)`);
462
+ return;
463
+ }
464
+ clog(`Conversation execution ${outcome} id=${eid ?? "none"}`);
465
+ settle(outcome);
311
466
  };
312
467
 
313
468
  unsubs = [
469
+ api.event.on("session.execution.started", (event) => {
470
+ const props = event?.properties || event?.data || {};
471
+ if (!forSession(props)) return;
472
+ const eid = executionIDOf(props);
473
+ if (eid) currentExecutionID = eid;
474
+ sawExecutionStart = true;
475
+ clog(`Conversation execution.started id=${eid ?? "none"}`);
476
+ }),
477
+ api.event.on("session.execution.succeeded", onExecutionEnd("idle")),
478
+ api.event.on("session.execution.failed", onExecutionEnd("idle")),
479
+ api.event.on("session.execution.interrupted", onExecutionEnd("stopped")),
480
+ // Legacy fallbacks, only trusted once this turn's assistant message
481
+ // actually exists with a terminal finish.
314
482
  api.event.on("session.idle", (event) => {
315
- if (forSession(event.properties)) finish("idle");
483
+ const props = event?.properties || event?.data || {};
484
+ if (!forSession(props)) return;
485
+ if (!turnFinished()) {
486
+ clog(`Conversation session.idle ignored finish=${lastFinish() ?? "none"}`);
487
+ return;
488
+ }
489
+ finish("idle");
316
490
  }),
317
491
  api.event.on("session.status", (event) => {
318
- if (event.properties?.status?.type === "idle" && forSession(event.properties)) {
319
- finish("idle");
492
+ const props = event?.properties || event?.data || {};
493
+ if (props?.status?.type !== "idle" || !forSession(props)) return;
494
+ if (!turnFinished()) {
495
+ clog(`Conversation session.status idle ignored finish=${lastFinish() ?? "none"}`);
496
+ return;
320
497
  }
498
+ finish("idle");
321
499
  }),
322
500
  ...(gates
323
501
  ? [
@@ -331,23 +509,57 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
331
509
  : []),
332
510
  ];
333
511
  timer = setTimeout(() => finish("timeout"), replyTimeoutMs);
512
+ // Last resort when the host emits no execution event at all.
513
+ pollTimer = setInterval(() => {
514
+ if (settled) return;
515
+ try {
516
+ const fr = lastFinish();
517
+ clog(`Conversation poll new=${newAssistants().length} finish=${fr ?? "none"}`);
518
+ if (isTerminal(fr)) settle("idle");
519
+ } catch {}
520
+ }, 700);
521
+ if (typeof pollTimer.unref === "function") pollTimer.unref();
334
522
  });
335
523
 
336
524
  return promise;
337
525
  }
338
526
 
339
527
  async function finishTurnFlow() {
528
+ clog(
529
+ `Conversation finishTurnFlow called active=${active} phase=${phase} streaming=${stt.isStreaming?.()}`,
530
+ );
340
531
  if (!active || phase !== "recording") return;
532
+ stt.setSilenceAutoFinalize?.(null, null);
533
+ stt.setConversationStreamingFinalize?.(null, null);
341
534
  const myGen = generation;
342
535
  phase = "processing";
536
+ convSetActions({
537
+ cancel: () => {
538
+ // Processing is too short to cancel safely; report busy.
539
+ },
540
+ });
541
+ convSetPhase("processing");
343
542
  showProcessingToast("Transcribing...");
344
543
 
345
- const res = await stt.transcribeTurn();
544
+ let res = null;
545
+ if (stt.isStreaming?.()) {
546
+ try {
547
+ res = await stt.finalizeStreaming?.({ submit: false, insert: false });
548
+ } catch (e) {
549
+ res = { error: e };
550
+ }
551
+ } else {
552
+ res = await stt.transcribeTurn();
553
+ }
346
554
  if (!active || myGen !== generation) return;
347
555
 
556
+ if (!res) {
557
+ pauseForRetry(`Paused - press ${keyLabel()} to retry`);
558
+ return;
559
+ }
348
560
  if (res.error) {
349
561
  consecErrors += 1;
350
- logger?.log("VOICE", `Conversation turn error consec=${consecErrors}`, "warn");
562
+ clog(`Conversation turn error consec=${consecErrors}`);
351
563
  if (consecErrors >= 2) {
352
564
  stop("error");
353
565
  return;
@@ -355,7 +567,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
355
567
  pauseForRetry(`Paused - press ${keyLabel()} to retry`);
356
568
  return;
357
569
  }
358
- if (!res.text) {
570
+ if (!res.text || res.text.trim().length < 3) {
359
571
  consecErrors = 0;
360
572
  pauseForRetry(`No speech heard - press ${keyLabel()} to try again`);
361
573
  return;
@@ -369,14 +581,32 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
369
581
  return;
370
582
  }
371
583
 
372
- const turnStart = Date.now();
584
+ // Baseline: how many messages the session already holds BEFORE submit.
585
+ // Only messages after this index belong to this turn. A session created by
586
+ // the submit itself did not exist yet, so its baseline is 0.
587
+ const baselineSessionID = currentSessionID();
588
+ let baselineCount = 0;
589
+ if (baselineSessionID) {
590
+ try {
591
+ baselineCount = (api?.state?.session?.messages?.(baselineSessionID) || []).length;
592
+ } catch {
593
+ baselineCount = 0;
594
+ }
595
+ }
596
+ const baselineFor = (sid) => (sid && sid === baselineSessionID ? baselineCount : 0);
597
+ clog(`Conversation baseline sid=${baselineSessionID ?? "new"} count=${baselineCount}`);
598
+
373
599
  // v2 prompt dispatch may emit the entire reply before its promise returns.
374
600
  // Home creation navigates before prompting, so resolve the ID at event time.
375
601
  let earlyWait = null;
376
602
  if (api.client?.tui?.v2) {
377
603
  phase = "waiting";
378
- startReplyStream(currentSessionID, turnStart);
379
- earlyWait = waitForReply(currentSessionID);
604
+ convSetActions({
605
+ cancel: () => {},
606
+ barge: () => bargeIn(),
607
+ });
608
+ convSetPhase("waiting");
609
+ earlyWait = waitForReply(currentSessionID, { baseline: baselineFor });
380
610
  }
381
611
  const submitted = await stt.submitTurnText(res.text);
382
612
  if (!active || myGen !== generation) return;
@@ -386,18 +616,18 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
386
616
  }
387
617
 
388
618
  phase = "waiting";
619
+ convSetActions({
620
+ cancel: () => {},
621
+ barge: () => bargeIn(),
622
+ });
623
+ convSetPhase("waiting");
389
624
  const sessionID = currentSessionID();
390
625
  showProcessingToast(`Working - press ${keyLabel()} to talk`);
391
- if (!earlyWait) startReplyStream(sessionID, turnStart);
392
626
  const tWait = Date.now();
393
- const outcome = await (earlyWait || waitForReply(sessionID));
627
+ const outcome = await (earlyWait || waitForReply(sessionID, { baseline: baselineFor }));
394
628
  const waitMs = Date.now() - tWait;
395
629
  endReplyStream();
396
- logger?.log(
397
- "VOICE",
398
- `Conversation wait outcome=${outcome} waitMs=${waitMs} streamedChars=${streamSpokenChars}`,
399
- "debug",
400
- );
630
+ logger?.log("VOICE", `Conversation wait outcome=${outcome} waitMs=${waitMs}`, "debug");
401
631
  if (!active || myGen !== generation) return;
402
632
 
403
633
  if (outcome === "timeout") {
@@ -407,7 +637,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
407
637
  return;
408
638
  }
409
639
  if (outcome === "permission" || outcome === "question") {
410
- await answerGateThenSpeak(myGen, sessionID, turnStart, outcome);
640
+ await answerGateThenSpeak(myGen, sessionID, baselineFor, outcome);
411
641
  return;
412
642
  }
413
643
  if (outcome === "stopped") return;
@@ -420,9 +650,27 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
420
650
  // agent to resume so its post-answer reply is spoken aloud - before this,
421
651
  // endReplyStream() had already run and the answer was dropped in silence.
422
652
 
423
- async function answerGateThenSpeak(myGen, sessionID, turnStart, outcome) {
653
+ async function answerGateThenSpeak(myGen, sessionID, baseline, outcome) {
424
654
  discardStreamAudio();
425
655
  phase = "speaking";
656
+ convSetActions({
657
+ stop: () => {
658
+ if (phase === "speaking") {
659
+ tts.stop();
660
+ phase = "paused";
661
+ convSetPhase("paused");
662
+ }
663
+ },
664
+ barge: () => bargeIn(),
665
+ pauseTts: () => {
666
+ if (phase === "speaking") {
667
+ tts.stop();
668
+ phase = "paused";
669
+ convSetPhase("paused");
670
+ }
671
+ },
672
+ });
673
+ convSetPhase("speaking");
426
674
  showProcessingToast("Speaking...");
427
675
  await tts.speak(
428
676
  outcome === "permission"
@@ -435,9 +683,13 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
435
683
  // Still "waiting": the user answers on screen, and a press here is an
436
684
  // ordinary barge-in (drop the pending reply, open the mic).
437
685
  phase = "waiting";
686
+ convSetActions({
687
+ cancel: () => {},
688
+ barge: () => bargeIn(),
689
+ });
690
+ convSetPhase("waiting");
438
691
  showProcessingToast(`Answer on screen - press ${keyLabel()} to talk`);
439
- startReplyStream(sessionID, turnStart);
440
- const resumed = await waitForReply(sessionID, { gates: false });
692
+ const resumed = await waitForReply(sessionID, { gates: false, baseline });
441
693
  if (!active || myGen !== generation) return;
442
694
  if (resumed === "stopped") return;
443
695
  endReplyStream();
@@ -451,24 +703,35 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
451
703
  }
452
704
 
453
705
  // Speak whatever this turn produced, then rest paused with the mic OFF.
454
- // Streamed sentences are already queued, so only the tail needs flushing;
455
- // the full LLM-narrated speak is the fallback when nothing streamable
456
- // arrived. Returns early on any generation/phase change (barge-in, stop).
706
+ // Only the completed turn is voiced: live deltas during tool calls are
707
+ // commentary, not the answer, and speaking them reads half a thought aloud.
708
+ // Returns early on any generation/phase change (barge-in, stop).
457
709
 
458
710
  async function speakReply(myGen) {
711
+ clog(`Conversation speakReply entry myGen=${myGen} gen=${generation} phase=${phase}`);
459
712
  phase = "speaking";
713
+ convSetActions({
714
+ stop: () => {
715
+ if (phase === "speaking") {
716
+ tts.stop();
717
+ phase = "paused";
718
+ convSetPhase("paused");
719
+ }
720
+ },
721
+ barge: () => bargeIn(),
722
+ pauseTts: () => {
723
+ if (phase === "speaking") {
724
+ tts.stop();
725
+ phase = "paused";
726
+ convSetPhase("paused");
727
+ }
728
+ },
729
+ });
730
+ convSetPhase("speaking");
460
731
  showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
461
- if (streamSpokenChars > 0) {
462
- flushStream(true);
463
- await speakTail;
464
- clearProcessingToast();
465
- if (!active || myGen !== generation || phase !== "speaking") return;
466
- await delay(restartDelayMs);
467
- if (!active || myGen !== generation || phase !== "speaking") return;
468
- enterPaused();
469
- return;
470
- }
732
+ clog("Conversation: calling tts.speakAssistantTurn");
471
733
  const spoken = await tts.speakAssistantTurn();
734
+ clog(`Conversation: speakAssistantTurn returned spoken=${spoken?.spoken}`);
472
735
  if (!active || myGen !== generation || phase !== "speaking") return;
473
736
  // Drop the sticky speaking status before the paused toast, or it keeps
474
737
  // re-announcing "interrupt" once the mic is closed again.
@@ -479,6 +742,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
479
742
  await delay(restartDelayMs);
480
743
  if (!active || myGen !== generation || phase !== "speaking") return;
481
744
  enterPaused();
745
+ if (!spoken.spoken) return;
746
+ // Hands-free: auto-listen after a successful reply.
747
+ await delay(500);
748
+ if (!active || myGen !== generation || phase !== "paused") return;
749
+ beginTurn();
482
750
  }
483
751
 
484
752
  // Pause instead of auto-recording so an empty or failed turn never feels
@@ -495,7 +763,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
495
763
 
496
764
  function pauseSpeech() {
497
765
  if (!active || phase !== "speaking") return;
498
- logger?.log("VOICE", "Conversation speech paused", "debug");
766
+ clog("Conversation speech paused");
499
767
  tts.stop();
500
768
  phase = "paused";
501
769
  toast(`Speech paused - press ${keyLabel()} to talk`);
@@ -560,9 +828,17 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
560
828
  consecErrors = 0;
561
829
  phase = "paused";
562
830
  tts.setConversationActive(true);
831
+ convBegin();
832
+ convSetActions({
833
+ talk: () => {
834
+ if (phase === "paused") beginTurn();
835
+ },
836
+ end: () => stop("user"),
837
+ });
838
+ convSetPhase("paused");
563
839
  // Recording toasts must name the key the user actually has bound.
564
840
  stt.setStopHint(kb("voice.conversation"));
565
- logger?.log("VOICE", "Conversation started", "debug");
841
+ clog("Conversation started");
566
842
  toast(`Conversation on - press ${keyLabel()} to talk`);
567
843
  }
568
844