@bojackduy/opencode-voice 0.11.3 → 0.11.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -15
- package/index.js +8 -1
- package/lib/chatterbox-server.js +9 -1
- package/lib/conversation-status.js +274 -0
- package/lib/conversation.js +392 -116
- package/lib/streaming-stt.js +81 -0
- package/lib/streaming-transcript.js +62 -6
- package/lib/stt.js +356 -34
- package/lib/tts-player.js +3 -1
- package/lib/tts.js +335 -32
- package/lib/v2-compat.js +22 -0
- package/lib/v2-preview.js +123 -23
- package/lib/vieneu-server.js +305 -0
- package/package.json +17 -3
package/lib/conversation.js
CHANGED
|
@@ -23,13 +23,11 @@
|
|
|
23
23
|
// or the global /voice-cancel. Empty or failed turns pause instead of
|
|
24
24
|
// auto-recording, so the key never surprises.
|
|
25
25
|
//
|
|
26
|
-
// Latency:
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
// LLM-narrated speak.
|
|
26
|
+
// Latency vs. honesty: the turn is voiced only after the whole execution
|
|
27
|
+
// (tool calls included) completes, so what is read aloud is the answer and
|
|
28
|
+
// never mid-turn commentary.
|
|
30
29
|
|
|
31
30
|
import { clearProcessingToast, showProcessingToast } from "./stt.js";
|
|
32
|
-
import { isSpeakableSentence, localSpeechCleanup, splitSpokenSentences } from "./tts.js";
|
|
33
31
|
|
|
34
32
|
export const DEFAULT_STOP_PHRASES = [
|
|
35
33
|
"stop",
|
|
@@ -105,28 +103,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
105
103
|
let active = false;
|
|
106
104
|
let phase = "idle"; // idle | recording | processing | waiting | speaking | paused
|
|
107
105
|
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
// the
|
|
111
|
-
|
|
112
|
-
let streamBuffer = "";
|
|
113
|
-
let streamSpokenChars = 0;
|
|
106
|
+
// Event subscriptions owned by the current wait/probe set. Nothing is
|
|
107
|
+
// spoken from live deltas: the agent's commentary during tool calls is not
|
|
108
|
+
// the answer, so only the completed turn is voiced (see speakReply).
|
|
114
109
|
let streamUnsubs = [];
|
|
115
|
-
let speakTail = Promise.resolve();
|
|
116
110
|
let speakGen = 0;
|
|
117
111
|
|
|
118
|
-
function queueSpeak(sentence) {
|
|
119
|
-
const g = speakGen;
|
|
120
|
-
speakTail = speakTail.then(() => {
|
|
121
|
-
// Speaking must also drain: finishTurnFlow flips to speaking before
|
|
122
|
-
// flushing the tail, so a waiting-only guard would skip queued
|
|
123
|
-
// sentences/final partials under real TTS latency. speakGen still
|
|
124
|
-
// invalidates on pause/stop.
|
|
125
|
-
if (g !== speakGen || !active || (phase !== "waiting" && phase !== "speaking")) return null;
|
|
126
|
-
return tts.speak(sentence);
|
|
127
|
-
});
|
|
128
|
-
return speakTail;
|
|
129
|
-
}
|
|
130
112
|
let turn = 0;
|
|
131
113
|
let consecErrors = 0;
|
|
132
114
|
let generation = 0;
|
|
@@ -136,6 +118,44 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
136
118
|
api.ui.toast({ message, variant, duration: 3000 });
|
|
137
119
|
}
|
|
138
120
|
|
|
121
|
+
// v2 persistent status line: phase-aware buttons. v1 has no surface, so
|
|
122
|
+
// these are no-ops there (api.conversationStatus is undefined).
|
|
123
|
+
const convStatus = api.conversationStatus || null;
|
|
124
|
+
const clog = (m) => {
|
|
125
|
+
try {
|
|
126
|
+
stt.log?.(m);
|
|
127
|
+
} catch {}
|
|
128
|
+
};
|
|
129
|
+
const setConvPhase = api.setConvPhase || (() => {});
|
|
130
|
+
const setConvActions = api.setConvActions || (() => {});
|
|
131
|
+
function convBegin() {
|
|
132
|
+
try {
|
|
133
|
+
const p = convStatus?.begin?.();
|
|
134
|
+
if (p && typeof p.then === "function") p.catch(() => {});
|
|
135
|
+
} catch {}
|
|
136
|
+
}
|
|
137
|
+
function convUpdate() {
|
|
138
|
+
try {
|
|
139
|
+
convStatus?.update?.();
|
|
140
|
+
} catch {}
|
|
141
|
+
}
|
|
142
|
+
function convClear() {
|
|
143
|
+
try {
|
|
144
|
+
convStatus?.clear?.();
|
|
145
|
+
} catch {}
|
|
146
|
+
}
|
|
147
|
+
function convSetPhase(p) {
|
|
148
|
+
try {
|
|
149
|
+
setConvPhase(p);
|
|
150
|
+
convUpdate();
|
|
151
|
+
} catch {}
|
|
152
|
+
}
|
|
153
|
+
function convSetActions(actions) {
|
|
154
|
+
try {
|
|
155
|
+
setConvActions(actions);
|
|
156
|
+
} catch {}
|
|
157
|
+
}
|
|
158
|
+
|
|
139
159
|
function currentSessionID() {
|
|
140
160
|
const route = api?.route?.current;
|
|
141
161
|
return route?.name === "session" ? route?.params?.sessionID : null;
|
|
@@ -155,14 +175,87 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
155
175
|
if (stt.isRecording() || stt.isProcessing() || stt.isStreaming?.()) return;
|
|
156
176
|
turn += 1;
|
|
157
177
|
phase = "recording";
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
178
|
+
clog(`Conversation turn ${turn} listening`);
|
|
179
|
+
const streamingMode = (api?.kv?.get?.("stt.mode") ?? opts?.sttMode) === "streaming";
|
|
180
|
+
// Allow streaming STT during conversation for waveform + live transcript.
|
|
181
|
+
try {
|
|
182
|
+
stt.setConversationStreamingEnabled?.(true);
|
|
183
|
+
} catch {}
|
|
184
|
+
const autoEnd = api?.kv?.get?.("stt.autoEndOfSpeech") === true;
|
|
185
|
+
if (streamingMode) {
|
|
186
|
+
// Register the stop callback even when auto-end is off: the preview
|
|
187
|
+
// toggle can turn auto-end on mid-recording, and the STT timer must
|
|
188
|
+
// then finish this turn through the conversation loop instead of
|
|
189
|
+
// submitting the text as plain dictation (which skips the reply wait,
|
|
190
|
+
// the speech, and leaves the UI on "Listening").
|
|
191
|
+
const onStreamAutoStop = () => {
|
|
192
|
+
clog(`Conversation streaming auto-finalize callback active=${active} phase=${phase}`);
|
|
193
|
+
if (active && phase === "recording") {
|
|
194
|
+
clog("Conversation auto-finalize on silence (streaming)");
|
|
195
|
+
void finishTurnFlow();
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
stt.setConversationStreamingFinalize?.(autoEnd ? 2.5 : null, onStreamAutoStop);
|
|
199
|
+
const started = stt.startStreaming?.();
|
|
200
|
+
if (!started) {
|
|
201
|
+
// Fallback to batch recording.
|
|
202
|
+
stt.setConversationStreamingFinalize?.(null, null);
|
|
203
|
+
if (autoEnd) {
|
|
204
|
+
stt.setSilenceAutoFinalize?.(2.5, () => {
|
|
205
|
+
if (active && phase === "recording") {
|
|
206
|
+
clog("Conversation auto-finalize on silence");
|
|
207
|
+
finishTurnFlow();
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
} else {
|
|
211
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
212
|
+
}
|
|
213
|
+
if (!stt.start()) {
|
|
214
|
+
phase = "idle";
|
|
215
|
+
stop("busy");
|
|
216
|
+
toast("STT busy, conversation off", "warning");
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
} else {
|
|
221
|
+
if (autoEnd) {
|
|
222
|
+
stt.setSilenceAutoFinalize?.(2.5, () => {
|
|
223
|
+
if (active && phase === "recording") {
|
|
224
|
+
clog("Conversation auto-finalize on silence");
|
|
225
|
+
finishTurnFlow();
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
} else {
|
|
229
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
230
|
+
}
|
|
231
|
+
if (!stt.start()) {
|
|
232
|
+
phase = "idle";
|
|
233
|
+
stop("busy");
|
|
234
|
+
toast("STT busy, conversation off", "warning");
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
164
237
|
}
|
|
165
|
-
|
|
238
|
+
convSetActions({
|
|
239
|
+
cancel: () => {
|
|
240
|
+
if (phase === "recording") {
|
|
241
|
+
try {
|
|
242
|
+
stt.cancelStreaming?.();
|
|
243
|
+
} catch {}
|
|
244
|
+
stt.cancel();
|
|
245
|
+
phase = "paused";
|
|
246
|
+
convSetPhase("paused");
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
send: () => {
|
|
250
|
+
if (phase === "recording") finishTurnFlow();
|
|
251
|
+
},
|
|
252
|
+
});
|
|
253
|
+
convSetPhase("recording");
|
|
254
|
+
toast(
|
|
255
|
+
autoEnd
|
|
256
|
+
? `Listening - speak, then pause 2.5s to send (or press ${keyLabel()})`
|
|
257
|
+
: `Listening - press ${keyLabel()} to send`,
|
|
258
|
+
);
|
|
166
259
|
}
|
|
167
260
|
|
|
168
261
|
// The resting state: mic closed, one press to talk. Entered by start() and
|
|
@@ -170,6 +263,13 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
170
263
|
|
|
171
264
|
function enterPaused() {
|
|
172
265
|
phase = "paused";
|
|
266
|
+
convSetActions({
|
|
267
|
+
talk: () => {
|
|
268
|
+
if (phase === "paused") beginTurn();
|
|
269
|
+
},
|
|
270
|
+
end: () => stop("user"),
|
|
271
|
+
});
|
|
272
|
+
convSetPhase("paused");
|
|
173
273
|
toast(`Press ${keyLabel()} to talk`);
|
|
174
274
|
}
|
|
175
275
|
|
|
@@ -178,7 +278,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
178
278
|
// means "I want to talk now", so it must never exit silently.
|
|
179
279
|
|
|
180
280
|
function bargeIn() {
|
|
181
|
-
|
|
281
|
+
clog("Conversation barge-in");
|
|
282
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
283
|
+
stt.setConversationStreamingFinalize?.(null, null);
|
|
284
|
+
try {
|
|
285
|
+
stt.cancelStreaming?.();
|
|
286
|
+
} catch {}
|
|
182
287
|
discardStreamAudio();
|
|
183
288
|
clearProcessingToast();
|
|
184
289
|
// Settle the pending reply wait as "stopped" so the abandoned answer is
|
|
@@ -214,6 +319,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
214
319
|
active = false;
|
|
215
320
|
generation += 1;
|
|
216
321
|
phase = "idle";
|
|
322
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
323
|
+
stt.setConversationStreamingFinalize?.(null, null);
|
|
324
|
+
try {
|
|
325
|
+
stt.setConversationStreamingEnabled?.(false);
|
|
326
|
+
} catch {}
|
|
217
327
|
if (waitCancel) {
|
|
218
328
|
waitCancel("stopped");
|
|
219
329
|
waitCancel = null;
|
|
@@ -221,71 +331,66 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
221
331
|
endReplyStream();
|
|
222
332
|
speakGen += 1;
|
|
223
333
|
stt.setStopHint(null);
|
|
334
|
+
try {
|
|
335
|
+
stt.cancelStreaming?.();
|
|
336
|
+
} catch {}
|
|
224
337
|
if (stt.isRecording() || stt.isProcessing()) stt.cancel();
|
|
225
338
|
clearProcessingToast();
|
|
226
339
|
tts.stop();
|
|
227
340
|
tts.setConversationActive(false);
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
function startReplyStream(sessionID, turnStart) {
|
|
232
|
-
streamBuffer = "";
|
|
233
|
-
streamSpokenChars = 0;
|
|
234
|
-
const unsub = api.event.on("message.part.delta", (event) => {
|
|
235
|
-
try {
|
|
236
|
-
if (!active || phase !== "waiting") return;
|
|
237
|
-
const props = event.properties || {};
|
|
238
|
-
const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
|
|
239
|
-
if (typeof sessionID === "function" && !expectedID) return;
|
|
240
|
-
if (expectedID && props.sessionID && props.sessionID !== expectedID) return;
|
|
241
|
-
if (typeof props.delta !== "string" || !props.delta) return;
|
|
242
|
-
if (props.field && props.field !== "text") return;
|
|
243
|
-
const lookupID = props.sessionID || expectedID;
|
|
244
|
-
const messages = api?.state?.session?.messages?.(lookupID) || [];
|
|
245
|
-
const msg = messages.find((m) => m?.id === props.messageID);
|
|
246
|
-
if (!msg || msg.role !== "assistant") return;
|
|
247
|
-
if ((msg.time?.created || 0) < turnStart) return;
|
|
248
|
-
const parts = api?.state?.part?.(props.messageID) || [];
|
|
249
|
-
const part = parts.find((p) => p?.id === props.partID);
|
|
250
|
-
if (part && part.type !== "text") return;
|
|
251
|
-
streamBuffer += props.delta;
|
|
252
|
-
flushStream(false);
|
|
253
|
-
} catch {}
|
|
254
|
-
});
|
|
255
|
-
streamUnsubs.push(unsub);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
function flushStream(final = false) {
|
|
259
|
-
const { sentences, rest } = splitSpokenSentences(streamBuffer);
|
|
260
|
-
const ready = final && rest.trim() ? [...sentences, rest] : sentences;
|
|
261
|
-
if (ready.length > 0 && streamSpokenChars === 0) {
|
|
262
|
-
showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
|
|
263
|
-
}
|
|
264
|
-
for (const sentence of ready) {
|
|
265
|
-
if (!isSpeakableSentence(sentence)) continue;
|
|
266
|
-
const cleaned = localSpeechCleanup(sentence);
|
|
267
|
-
if (!cleaned) continue;
|
|
268
|
-
queueSpeak(cleaned);
|
|
269
|
-
streamSpokenChars += cleaned.length;
|
|
270
|
-
}
|
|
271
|
-
streamBuffer = final ? "" : rest;
|
|
341
|
+
convClear();
|
|
342
|
+
clog(`Conversation stopped reason=${reason}`);
|
|
272
343
|
}
|
|
273
344
|
|
|
274
345
|
// Wait for the submitted turn to finish: idle, a permission/question gate,
|
|
275
346
|
// or timeout. Resolves early when stop() cancels the wait. gates:false drops
|
|
276
347
|
// the permission/question subscriptions, so a second wait survives the gate
|
|
277
348
|
// the user is answering on screen and only ends on idle/timeout/stop.
|
|
278
|
-
|
|
279
|
-
|
|
349
|
+
//
|
|
350
|
+
// Completion is a host fact, not a guess: v2 emits
|
|
351
|
+
// `session.execution.succeeded|failed|interrupted` only when the whole
|
|
352
|
+
// execution - tool calls included - ends. The message list is the fallback
|
|
353
|
+
// signal, and BOTH paths are gated on `baseline` (the message count taken
|
|
354
|
+
// before submit), so a finished assistant message from an earlier turn can
|
|
355
|
+
// never be mistaken for this turn's reply. That gate is the fix for the
|
|
356
|
+
// reply being read ~800ms early with zero assistant messages in it.
|
|
357
|
+
|
|
358
|
+
function waitForReply(sessionID, { gates = true, baseline = 0 } = {}) {
|
|
280
359
|
let settled = false;
|
|
281
360
|
let unsubs = [];
|
|
282
361
|
let timer = null;
|
|
362
|
+
let pollTimer = null;
|
|
363
|
+
let settleTimer = null;
|
|
364
|
+
let currentExecutionID = null;
|
|
365
|
+
let sawExecutionStart = false;
|
|
366
|
+
|
|
367
|
+
const resolveSessionID = () => (typeof sessionID === "function" ? sessionID() : sessionID);
|
|
368
|
+
const baselineFor = (sid) => (typeof baseline === "function" ? baseline(sid) : baseline);
|
|
369
|
+
|
|
370
|
+
// Assistant messages that arrived AFTER submit, oldest first.
|
|
371
|
+
const newAssistants = () => {
|
|
372
|
+
try {
|
|
373
|
+
const sid = resolveSessionID();
|
|
374
|
+
const msgs = api?.state?.session?.messages?.(sid) || [];
|
|
375
|
+
return msgs.slice(baselineFor(sid)).filter((m) => m?.role === "assistant");
|
|
376
|
+
} catch {
|
|
377
|
+
return [];
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
const isTerminal = (finish) => !!finish && finish !== "tool-calls" && finish !== "tool_calls";
|
|
381
|
+
const lastFinish = () => {
|
|
382
|
+
const list = newAssistants();
|
|
383
|
+
return list.length ? list[list.length - 1]?.finish : null;
|
|
384
|
+
};
|
|
385
|
+
const turnFinished = () => isTerminal(lastFinish());
|
|
283
386
|
|
|
284
387
|
let cancelFn = null;
|
|
285
388
|
const done = (outcome) => {
|
|
286
389
|
if (settled) return null;
|
|
287
390
|
settled = true;
|
|
288
391
|
if (timer) clearTimeout(timer);
|
|
392
|
+
if (settleTimer) clearInterval(settleTimer);
|
|
393
|
+
if (pollTimer) clearInterval(pollTimer);
|
|
289
394
|
for (const unsub of unsubs) {
|
|
290
395
|
try {
|
|
291
396
|
unsub();
|
|
@@ -304,20 +409,93 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
304
409
|
cancelFn = (outcome) => finish(outcome || "stopped");
|
|
305
410
|
waitCancel = cancelFn;
|
|
306
411
|
|
|
412
|
+
// The host event can land a beat before the message cache does. Give the
|
|
413
|
+
// message at most 2s to appear, then settle anyway: the execution is
|
|
414
|
+
// over, and "no reply to speak" is a truthful outcome.
|
|
415
|
+
const settle = (outcome) => {
|
|
416
|
+
if (settled) return;
|
|
417
|
+
if (turnFinished()) {
|
|
418
|
+
finish(outcome);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
if (settleTimer) return;
|
|
422
|
+
const startedAt = Date.now();
|
|
423
|
+
settleTimer = setInterval(() => {
|
|
424
|
+
if (settled) return;
|
|
425
|
+
if (turnFinished() || Date.now() - startedAt > 2000) {
|
|
426
|
+
clog(`Conversation settled outcome=${outcome} finish=${lastFinish() ?? "none"}`);
|
|
427
|
+
finish(outcome);
|
|
428
|
+
}
|
|
429
|
+
}, 100);
|
|
430
|
+
if (typeof settleTimer.unref === "function") settleTimer.unref();
|
|
431
|
+
};
|
|
432
|
+
|
|
307
433
|
const forSession = (props) => {
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
-
|
|
434
|
+
const sid = props?.sessionID ?? props?.id ?? props?.session?.id;
|
|
435
|
+
const expectedID = resolveSessionID();
|
|
436
|
+
if (typeof sessionID === "function" && !expectedID) return true;
|
|
437
|
+
if (!expectedID) return true;
|
|
438
|
+
if (!sid) return true;
|
|
439
|
+
return sid === expectedID;
|
|
440
|
+
};
|
|
441
|
+
|
|
442
|
+
const executionIDOf = (props) =>
|
|
443
|
+
props?.executionID ?? props?.executionId ?? props?.id ?? props?.execution?.id;
|
|
444
|
+
|
|
445
|
+
const onExecutionEnd = (outcome) => (event) => {
|
|
446
|
+
const props = event?.properties || event?.data || {};
|
|
447
|
+
if (!forSession(props)) return;
|
|
448
|
+
const eid = executionIDOf(props);
|
|
449
|
+
if (currentExecutionID && eid && eid !== currentExecutionID) {
|
|
450
|
+
clog(
|
|
451
|
+
`Conversation execution ${outcome} ignored stale id=${eid} current=${currentExecutionID}`,
|
|
452
|
+
);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
// The host's execution events carry no execution id, so the guard is
|
|
456
|
+
// structural: a terminal event is only trusted once this wait has seen
|
|
457
|
+
// execution.started, or once this turn's own assistant message is
|
|
458
|
+
// there with a terminal finish. Otherwise it is a late event from the
|
|
459
|
+
// turn that barge-in interrupted.
|
|
460
|
+
if (!sawExecutionStart && !turnFinished()) {
|
|
461
|
+
clog(`Conversation execution ${outcome} ignored (no start, no new assistant)`);
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
clog(`Conversation execution ${outcome} id=${eid ?? "none"}`);
|
|
465
|
+
settle(outcome);
|
|
311
466
|
};
|
|
312
467
|
|
|
313
468
|
unsubs = [
|
|
469
|
+
api.event.on("session.execution.started", (event) => {
|
|
470
|
+
const props = event?.properties || event?.data || {};
|
|
471
|
+
if (!forSession(props)) return;
|
|
472
|
+
const eid = executionIDOf(props);
|
|
473
|
+
if (eid) currentExecutionID = eid;
|
|
474
|
+
sawExecutionStart = true;
|
|
475
|
+
clog(`Conversation execution.started id=${eid ?? "none"}`);
|
|
476
|
+
}),
|
|
477
|
+
api.event.on("session.execution.succeeded", onExecutionEnd("idle")),
|
|
478
|
+
api.event.on("session.execution.failed", onExecutionEnd("idle")),
|
|
479
|
+
api.event.on("session.execution.interrupted", onExecutionEnd("stopped")),
|
|
480
|
+
// Legacy fallbacks, only trusted once this turn's assistant message
|
|
481
|
+
// actually exists with a terminal finish.
|
|
314
482
|
api.event.on("session.idle", (event) => {
|
|
315
|
-
|
|
483
|
+
const props = event?.properties || event?.data || {};
|
|
484
|
+
if (!forSession(props)) return;
|
|
485
|
+
if (!turnFinished()) {
|
|
486
|
+
clog(`Conversation session.idle ignored finish=${lastFinish() ?? "none"}`);
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
finish("idle");
|
|
316
490
|
}),
|
|
317
491
|
api.event.on("session.status", (event) => {
|
|
318
|
-
|
|
319
|
-
|
|
492
|
+
const props = event?.properties || event?.data || {};
|
|
493
|
+
if (props?.status?.type !== "idle" || !forSession(props)) return;
|
|
494
|
+
if (!turnFinished()) {
|
|
495
|
+
clog(`Conversation session.status idle ignored finish=${lastFinish() ?? "none"}`);
|
|
496
|
+
return;
|
|
320
497
|
}
|
|
498
|
+
finish("idle");
|
|
321
499
|
}),
|
|
322
500
|
...(gates
|
|
323
501
|
? [
|
|
@@ -331,23 +509,57 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
331
509
|
: []),
|
|
332
510
|
];
|
|
333
511
|
timer = setTimeout(() => finish("timeout"), replyTimeoutMs);
|
|
512
|
+
// Last resort when the host emits no execution event at all.
|
|
513
|
+
pollTimer = setInterval(() => {
|
|
514
|
+
if (settled) return;
|
|
515
|
+
try {
|
|
516
|
+
const fr = lastFinish();
|
|
517
|
+
clog(`Conversation poll new=${newAssistants().length} finish=${fr ?? "none"}`);
|
|
518
|
+
if (isTerminal(fr)) settle("idle");
|
|
519
|
+
} catch {}
|
|
520
|
+
}, 700);
|
|
521
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
334
522
|
});
|
|
335
523
|
|
|
336
524
|
return promise;
|
|
337
525
|
}
|
|
338
526
|
|
|
339
527
|
async function finishTurnFlow() {
|
|
528
|
+
clog(
|
|
529
|
+
`Conversation finishTurnFlow called active=${active} phase=${phase} streaming=${stt.isStreaming?.()}`,
|
|
530
|
+
);
|
|
340
531
|
if (!active || phase !== "recording") return;
|
|
532
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
533
|
+
stt.setConversationStreamingFinalize?.(null, null);
|
|
341
534
|
const myGen = generation;
|
|
342
535
|
phase = "processing";
|
|
536
|
+
convSetActions({
|
|
537
|
+
cancel: () => {
|
|
538
|
+
// Processing is too short to cancel safely; report busy.
|
|
539
|
+
},
|
|
540
|
+
});
|
|
541
|
+
convSetPhase("processing");
|
|
343
542
|
showProcessingToast("Transcribing...");
|
|
344
543
|
|
|
345
|
-
|
|
544
|
+
let res = null;
|
|
545
|
+
if (stt.isStreaming?.()) {
|
|
546
|
+
try {
|
|
547
|
+
res = await stt.finalizeStreaming?.({ submit: false, insert: false });
|
|
548
|
+
} catch (e) {
|
|
549
|
+
res = { error: e };
|
|
550
|
+
}
|
|
551
|
+
} else {
|
|
552
|
+
res = await stt.transcribeTurn();
|
|
553
|
+
}
|
|
346
554
|
if (!active || myGen !== generation) return;
|
|
347
555
|
|
|
556
|
+
if (!res) {
|
|
557
|
+
pauseForRetry(`Paused - press ${keyLabel()} to retry`);
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
348
560
|
if (res.error) {
|
|
349
561
|
consecErrors += 1;
|
|
350
|
-
|
|
562
|
+
clog(`Conversation turn error consec=${consecErrors}`);
|
|
351
563
|
if (consecErrors >= 2) {
|
|
352
564
|
stop("error");
|
|
353
565
|
return;
|
|
@@ -355,7 +567,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
355
567
|
pauseForRetry(`Paused - press ${keyLabel()} to retry`);
|
|
356
568
|
return;
|
|
357
569
|
}
|
|
358
|
-
if (!res.text) {
|
|
570
|
+
if (!res.text || res.text.trim().length < 3) {
|
|
359
571
|
consecErrors = 0;
|
|
360
572
|
pauseForRetry(`No speech heard - press ${keyLabel()} to try again`);
|
|
361
573
|
return;
|
|
@@ -369,14 +581,32 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
369
581
|
return;
|
|
370
582
|
}
|
|
371
583
|
|
|
372
|
-
|
|
584
|
+
// Baseline: how many messages the session already holds BEFORE submit.
|
|
585
|
+
// Only messages after this index belong to this turn. A session created by
|
|
586
|
+
// the submit itself did not exist yet, so its baseline is 0.
|
|
587
|
+
const baselineSessionID = currentSessionID();
|
|
588
|
+
let baselineCount = 0;
|
|
589
|
+
if (baselineSessionID) {
|
|
590
|
+
try {
|
|
591
|
+
baselineCount = (api?.state?.session?.messages?.(baselineSessionID) || []).length;
|
|
592
|
+
} catch {
|
|
593
|
+
baselineCount = 0;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
const baselineFor = (sid) => (sid && sid === baselineSessionID ? baselineCount : 0);
|
|
597
|
+
clog(`Conversation baseline sid=${baselineSessionID ?? "new"} count=${baselineCount}`);
|
|
598
|
+
|
|
373
599
|
// v2 prompt dispatch may emit the entire reply before its promise returns.
|
|
374
600
|
// Home creation navigates before prompting, so resolve the ID at event time.
|
|
375
601
|
let earlyWait = null;
|
|
376
602
|
if (api.client?.tui?.v2) {
|
|
377
603
|
phase = "waiting";
|
|
378
|
-
|
|
379
|
-
|
|
604
|
+
convSetActions({
|
|
605
|
+
cancel: () => {},
|
|
606
|
+
barge: () => bargeIn(),
|
|
607
|
+
});
|
|
608
|
+
convSetPhase("waiting");
|
|
609
|
+
earlyWait = waitForReply(currentSessionID, { baseline: baselineFor });
|
|
380
610
|
}
|
|
381
611
|
const submitted = await stt.submitTurnText(res.text);
|
|
382
612
|
if (!active || myGen !== generation) return;
|
|
@@ -386,18 +616,18 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
386
616
|
}
|
|
387
617
|
|
|
388
618
|
phase = "waiting";
|
|
619
|
+
convSetActions({
|
|
620
|
+
cancel: () => {},
|
|
621
|
+
barge: () => bargeIn(),
|
|
622
|
+
});
|
|
623
|
+
convSetPhase("waiting");
|
|
389
624
|
const sessionID = currentSessionID();
|
|
390
625
|
showProcessingToast(`Working - press ${keyLabel()} to talk`);
|
|
391
|
-
if (!earlyWait) startReplyStream(sessionID, turnStart);
|
|
392
626
|
const tWait = Date.now();
|
|
393
|
-
const outcome = await (earlyWait || waitForReply(sessionID));
|
|
627
|
+
const outcome = await (earlyWait || waitForReply(sessionID, { baseline: baselineFor }));
|
|
394
628
|
const waitMs = Date.now() - tWait;
|
|
395
629
|
endReplyStream();
|
|
396
|
-
logger?.log(
|
|
397
|
-
"VOICE",
|
|
398
|
-
`Conversation wait outcome=${outcome} waitMs=${waitMs} streamedChars=${streamSpokenChars}`,
|
|
399
|
-
"debug",
|
|
400
|
-
);
|
|
630
|
+
logger?.log("VOICE", `Conversation wait outcome=${outcome} waitMs=${waitMs}`, "debug");
|
|
401
631
|
if (!active || myGen !== generation) return;
|
|
402
632
|
|
|
403
633
|
if (outcome === "timeout") {
|
|
@@ -407,7 +637,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
407
637
|
return;
|
|
408
638
|
}
|
|
409
639
|
if (outcome === "permission" || outcome === "question") {
|
|
410
|
-
await answerGateThenSpeak(myGen, sessionID,
|
|
640
|
+
await answerGateThenSpeak(myGen, sessionID, baselineFor, outcome);
|
|
411
641
|
return;
|
|
412
642
|
}
|
|
413
643
|
if (outcome === "stopped") return;
|
|
@@ -420,9 +650,27 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
420
650
|
// agent to resume so its post-answer reply is spoken aloud - before this,
|
|
421
651
|
// endReplyStream() had already run and the answer was dropped in silence.
|
|
422
652
|
|
|
423
|
-
async function answerGateThenSpeak(myGen, sessionID,
|
|
653
|
+
async function answerGateThenSpeak(myGen, sessionID, baseline, outcome) {
|
|
424
654
|
discardStreamAudio();
|
|
425
655
|
phase = "speaking";
|
|
656
|
+
convSetActions({
|
|
657
|
+
stop: () => {
|
|
658
|
+
if (phase === "speaking") {
|
|
659
|
+
tts.stop();
|
|
660
|
+
phase = "paused";
|
|
661
|
+
convSetPhase("paused");
|
|
662
|
+
}
|
|
663
|
+
},
|
|
664
|
+
barge: () => bargeIn(),
|
|
665
|
+
pauseTts: () => {
|
|
666
|
+
if (phase === "speaking") {
|
|
667
|
+
tts.stop();
|
|
668
|
+
phase = "paused";
|
|
669
|
+
convSetPhase("paused");
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
});
|
|
673
|
+
convSetPhase("speaking");
|
|
426
674
|
showProcessingToast("Speaking...");
|
|
427
675
|
await tts.speak(
|
|
428
676
|
outcome === "permission"
|
|
@@ -435,9 +683,13 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
435
683
|
// Still "waiting": the user answers on screen, and a press here is an
|
|
436
684
|
// ordinary barge-in (drop the pending reply, open the mic).
|
|
437
685
|
phase = "waiting";
|
|
686
|
+
convSetActions({
|
|
687
|
+
cancel: () => {},
|
|
688
|
+
barge: () => bargeIn(),
|
|
689
|
+
});
|
|
690
|
+
convSetPhase("waiting");
|
|
438
691
|
showProcessingToast(`Answer on screen - press ${keyLabel()} to talk`);
|
|
439
|
-
|
|
440
|
-
const resumed = await waitForReply(sessionID, { gates: false });
|
|
692
|
+
const resumed = await waitForReply(sessionID, { gates: false, baseline });
|
|
441
693
|
if (!active || myGen !== generation) return;
|
|
442
694
|
if (resumed === "stopped") return;
|
|
443
695
|
endReplyStream();
|
|
@@ -451,24 +703,35 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
451
703
|
}
|
|
452
704
|
|
|
453
705
|
// Speak whatever this turn produced, then rest paused with the mic OFF.
|
|
454
|
-
//
|
|
455
|
-
// the
|
|
456
|
-
//
|
|
706
|
+
// Only the completed turn is voiced: live deltas during tool calls are
|
|
707
|
+
// commentary, not the answer, and speaking them reads half a thought aloud.
|
|
708
|
+
// Returns early on any generation/phase change (barge-in, stop).
|
|
457
709
|
|
|
458
710
|
async function speakReply(myGen) {
|
|
711
|
+
clog(`Conversation speakReply entry myGen=${myGen} gen=${generation} phase=${phase}`);
|
|
459
712
|
phase = "speaking";
|
|
713
|
+
convSetActions({
|
|
714
|
+
stop: () => {
|
|
715
|
+
if (phase === "speaking") {
|
|
716
|
+
tts.stop();
|
|
717
|
+
phase = "paused";
|
|
718
|
+
convSetPhase("paused");
|
|
719
|
+
}
|
|
720
|
+
},
|
|
721
|
+
barge: () => bargeIn(),
|
|
722
|
+
pauseTts: () => {
|
|
723
|
+
if (phase === "speaking") {
|
|
724
|
+
tts.stop();
|
|
725
|
+
phase = "paused";
|
|
726
|
+
convSetPhase("paused");
|
|
727
|
+
}
|
|
728
|
+
},
|
|
729
|
+
});
|
|
730
|
+
convSetPhase("speaking");
|
|
460
731
|
showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
|
|
461
|
-
|
|
462
|
-
flushStream(true);
|
|
463
|
-
await speakTail;
|
|
464
|
-
clearProcessingToast();
|
|
465
|
-
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
466
|
-
await delay(restartDelayMs);
|
|
467
|
-
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
468
|
-
enterPaused();
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
732
|
+
clog("Conversation: calling tts.speakAssistantTurn");
|
|
471
733
|
const spoken = await tts.speakAssistantTurn();
|
|
734
|
+
clog(`Conversation: speakAssistantTurn returned spoken=${spoken?.spoken}`);
|
|
472
735
|
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
473
736
|
// Drop the sticky speaking status before the paused toast, or it keeps
|
|
474
737
|
// re-announcing "interrupt" once the mic is closed again.
|
|
@@ -479,6 +742,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
479
742
|
await delay(restartDelayMs);
|
|
480
743
|
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
481
744
|
enterPaused();
|
|
745
|
+
if (!spoken.spoken) return;
|
|
746
|
+
// Hands-free: auto-listen after a successful reply.
|
|
747
|
+
await delay(500);
|
|
748
|
+
if (!active || myGen !== generation || phase !== "paused") return;
|
|
749
|
+
beginTurn();
|
|
482
750
|
}
|
|
483
751
|
|
|
484
752
|
// Pause instead of auto-recording so an empty or failed turn never feels
|
|
@@ -495,7 +763,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
495
763
|
|
|
496
764
|
function pauseSpeech() {
|
|
497
765
|
if (!active || phase !== "speaking") return;
|
|
498
|
-
|
|
766
|
+
clog("Conversation speech paused");
|
|
499
767
|
tts.stop();
|
|
500
768
|
phase = "paused";
|
|
501
769
|
toast(`Speech paused - press ${keyLabel()} to talk`);
|
|
@@ -560,9 +828,17 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
560
828
|
consecErrors = 0;
|
|
561
829
|
phase = "paused";
|
|
562
830
|
tts.setConversationActive(true);
|
|
831
|
+
convBegin();
|
|
832
|
+
convSetActions({
|
|
833
|
+
talk: () => {
|
|
834
|
+
if (phase === "paused") beginTurn();
|
|
835
|
+
},
|
|
836
|
+
end: () => stop("user"),
|
|
837
|
+
});
|
|
838
|
+
convSetPhase("paused");
|
|
563
839
|
// Recording toasts must name the key the user actually has bound.
|
|
564
840
|
stt.setStopHint(kb("voice.conversation"));
|
|
565
|
-
|
|
841
|
+
clog("Conversation started");
|
|
566
842
|
toast(`Conversation on - press ${keyLabel()} to talk`);
|
|
567
843
|
}
|
|
568
844
|
|