@jskit-ai/assistant-voice 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/examples/conversation/App.vue +2 -1
- package/examples/conversation/package.json +8 -8
- package/package.json +2 -2
- package/src/client/ConversationDialog.vue +23 -12
- package/src/client/VoiceConversation.vue +41 -15
- package/src/client/VoiceConversationControls.vue +102 -40
- package/src/client/VoiceConversationHost.vue +32 -31
- package/src/client/VoiceConversationSettings.vue +52 -0
- package/src/client/index.js +1 -0
- package/src/client/voiceConversation.js +223 -65
- package/src/client/voiceTransport.js +58 -10
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
<script setup>
|
|
2
|
+
import { mdiCog } from "@mdi/js";
|
|
3
|
+
import { useSlots } from "vue";
|
|
4
|
+
|
|
5
|
+
const props = defineProps({
|
|
6
|
+
voice: { type: Object, default: null },
|
|
7
|
+
compact: Boolean
|
|
8
|
+
});
|
|
9
|
+
const emit = defineEmits(["error"]);
|
|
10
|
+
const slots = useSlots();
|
|
11
|
+
function selectVoice(value) {
|
|
12
|
+
const selectedVoice = props.voice.selectedVoice;
|
|
13
|
+
selectedVoice.value = value;
|
|
14
|
+
}
|
|
15
|
+
async function openSettings(open) {
|
|
16
|
+
if (!open || slots.default || !props.voice) return;
|
|
17
|
+
try { await props.voice.connect(); }
|
|
18
|
+
catch (error) { emit("error", error); }
|
|
19
|
+
}
|
|
20
|
+
</script>
|
|
21
|
+
|
|
22
|
+
<template>
|
|
23
|
+
<v-menu :close-on-content-click="false" location="bottom end" @update:model-value="openSettings">
|
|
24
|
+
<template #activator="{ props: settingsButton }">
|
|
25
|
+
<v-btn
|
|
26
|
+
v-bind="settingsButton" :icon="mdiCog" variant="text" :color="compact ? 'primary' : undefined"
|
|
27
|
+
:width="compact ? 40 : undefined" :min-width="compact ? 40 : 48" :min-height="compact ? 44 : 48" :disabled="!voice && !$slots.default"
|
|
28
|
+
aria-label="Voice settings" title="Voice settings"
|
|
29
|
+
>
|
|
30
|
+
<template v-if="compact" #default><span class="voice-settings__disc"><v-icon size="20" :icon="mdiCog" /></span></template>
|
|
31
|
+
</v-btn>
|
|
32
|
+
</template>
|
|
33
|
+
<v-card class="voice-settings__menu" role="region" aria-label="Voice settings">
|
|
34
|
+
<v-card-text>
|
|
35
|
+
<slot :voice="voice">
|
|
36
|
+
<v-select
|
|
37
|
+
v-if="voice" :model-value="voice.selectedVoice.value" @update:model-value="selectVoice"
|
|
38
|
+
:items="voice.availableVoices.value" item-title="label" item-value="id"
|
|
39
|
+
:disabled="voice.availableVoices.value.length < 2"
|
|
40
|
+
label="Speaking voice" density="comfortable"
|
|
41
|
+
hint="Applies to the next spoken reply" persistent-hint
|
|
42
|
+
/>
|
|
43
|
+
</slot>
|
|
44
|
+
</v-card-text>
|
|
45
|
+
</v-card>
|
|
46
|
+
</v-menu>
|
|
47
|
+
</template>
|
|
48
|
+
|
|
49
|
+
<style scoped>
|
|
50
|
+
.voice-settings__menu { width: 320px; max-width: calc(100vw - 32px); }
|
|
51
|
+
.voice-settings__disc { display: inline-flex; align-items: center; justify-content: center; width: 32px; height: 32px; border-radius: 50%; background: rgba(var(--v-theme-primary), .12); }
|
|
52
|
+
</style>
|
package/src/client/index.js
CHANGED
|
@@ -4,6 +4,7 @@ export { default as VoiceConversationHost } from "./VoiceConversationHost.vue";
|
|
|
4
4
|
export { default as ConversationDialog } from "./ConversationDialog.vue";
|
|
5
5
|
export { default as VoiceConversation } from "./VoiceConversation.vue";
|
|
6
6
|
export { default as VoiceConversationControls } from "./VoiceConversationControls.vue";
|
|
7
|
+
export { default as VoiceConversationSettings } from "./VoiceConversationSettings.vue";
|
|
7
8
|
|
|
8
9
|
export { useVoiceLauncher } from "./voiceLauncher.js";
|
|
9
10
|
export { useVoiceAvatar } from "./voiceAvatar.js";
|
|
@@ -6,12 +6,16 @@ import { takeStreamingSpeech } from "../shared/protocol.js";
|
|
|
6
6
|
export function useVoiceConversation(binding, { socketUrl, createTransport = useVoiceTransport } = {}) {
|
|
7
7
|
const oneOffTalkMode = computed(() => binding.defaults?.talkMode === "hold" ? "hold" : "tap");
|
|
8
8
|
const targetLabel = computed(() => binding.label || "Assistant");
|
|
9
|
-
const readAloud = ref(
|
|
9
|
+
const readAloud = ref(binding.defaults?.readAloud === true);
|
|
10
|
+
const soundPreparing = ref(false);
|
|
11
|
+
let soundRevision = 0;
|
|
12
|
+
const conversationId = binding.conversationId || binding.id;
|
|
10
13
|
const error = ref("");
|
|
11
14
|
const pendingTranscript = ref(null);
|
|
12
15
|
const heldReview = ref(false);
|
|
13
16
|
const holding = ref(false);
|
|
14
17
|
const sending = ref(false);
|
|
18
|
+
const takingTranscript = ref(false);
|
|
15
19
|
const live = ref(false);
|
|
16
20
|
const starting = ref(false);
|
|
17
21
|
const callMode = ref(binding.defaults?.mode === "hands-free" ? "hands-free" : "push-to-talk");
|
|
@@ -27,7 +31,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
27
31
|
voiceId: binding.defaults?.voiceId || "",
|
|
28
32
|
speechEnabled: readAloud,
|
|
29
33
|
interruptSpeechOnListen: false,
|
|
30
|
-
autoReconnect: true
|
|
34
|
+
autoReconnect: true,
|
|
35
|
+
onPlayback: receivePlayback
|
|
31
36
|
});
|
|
32
37
|
watch([() => binding.defaults?.voiceId, voice.availableVoices], ([voiceId, voices]) => {
|
|
33
38
|
if (voiceId === undefined) return;
|
|
@@ -106,12 +111,42 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
106
111
|
let speechInvitation = "";
|
|
107
112
|
let lastStreamingId = "";
|
|
108
113
|
|
|
109
|
-
function
|
|
110
|
-
|
|
114
|
+
function observe(callback, value) {
|
|
115
|
+
try {
|
|
116
|
+
Promise.resolve(callback?.(value)).catch(() => {});
|
|
117
|
+
} catch {
|
|
118
|
+
// Observers cannot take ownership of scheduling or cleanup.
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
function emitReply(reply, phase, reason) {
|
|
122
|
+
if (reply.terminal || (phase === "started" && reply.audible)) return;
|
|
123
|
+
if (phase === "started") reply.audible = true;
|
|
124
|
+
else reply.terminal = true;
|
|
125
|
+
observe(binding.onPlayback, {
|
|
126
|
+
conversationId, outputId: reply.outputId, phase, ...(reason ? { reason } : {})
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
function receivePlayback(event) {
|
|
130
|
+
const reply = [...replies.values()].find(reply => reply.turnId === event.turnId);
|
|
131
|
+
if (!reply) return;
|
|
132
|
+
if (event.phase === "completed") {
|
|
133
|
+
reply.drained = true;
|
|
134
|
+
if (reply.canonicalFinal) emitReply(reply, "completed");
|
|
135
|
+
} else emitReply(reply, event.phase, event.reason);
|
|
136
|
+
}
|
|
137
|
+
function cancelReply(reply, reason = "cancelled", phase = "interrupted") {
|
|
138
|
+
reply.cancelled = true;
|
|
139
|
+
emitReply(reply, phase, reason);
|
|
140
|
+
}
|
|
141
|
+
function stopSpeech() { cancelSpeech(); }
|
|
142
|
+
function cancelSpeech(reason = "cancelled", phase = "interrupted") {
|
|
143
|
+
for (const reply of replies.values()) {
|
|
144
|
+
if (!reply.terminal) cancelReply(reply, reason, phase);
|
|
145
|
+
}
|
|
111
146
|
speechQueue.length = 0;
|
|
112
147
|
speechSuppressed = true;
|
|
113
148
|
speechInvitation = "";
|
|
114
|
-
voice.stopSpeaking();
|
|
149
|
+
voice.stopSpeaking(reason);
|
|
115
150
|
}
|
|
116
151
|
function inviteSpeech(messageId) {
|
|
117
152
|
// Only a new request from this page can lift silence. A delayed admission,
|
|
@@ -121,7 +156,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
121
156
|
speechInvitation = messageId;
|
|
122
157
|
}
|
|
123
158
|
async function drainSpeech() {
|
|
124
|
-
if (draining || disposed || !readAloud.value || speechSuppressed) return;
|
|
159
|
+
if (draining || disposed || !readAloud.value || speechSuppressed || soundPreparing.value || voice.playbackBlocked?.value) return;
|
|
125
160
|
draining = true;
|
|
126
161
|
try {
|
|
127
162
|
while (speechQueue.length) {
|
|
@@ -133,15 +168,24 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
133
168
|
const chunk = takeStreamingSpeech(reply.raw.slice(reply.consumed), reply.final, reply.started ? 140 : 64);
|
|
134
169
|
if (!chunk) {
|
|
135
170
|
if (reply.final && reply.started) voice.endSpeech();
|
|
136
|
-
else if (reply.final) {
|
|
171
|
+
else if (reply.final) {
|
|
172
|
+
cancelReply(reply, "no-audio");
|
|
173
|
+
speechQueue.shift();
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
137
176
|
return;
|
|
138
177
|
}
|
|
139
178
|
reply.consumed += chunk.consumed;
|
|
140
179
|
if (!chunk.text) continue;
|
|
141
180
|
if (!reply.started) {
|
|
142
181
|
reply.started = true;
|
|
143
|
-
|
|
182
|
+
reply.turnId = crypto.randomUUID();
|
|
183
|
+
const accepted = await voice.speak(chunk.text, reply.turnId, { stream: true });
|
|
144
184
|
if (reply.cancelled || disposed) return;
|
|
185
|
+
if (accepted === false) {
|
|
186
|
+
cancelReply(reply, "no-audio");
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
145
189
|
} else if (!voice.appendSpeech(chunk.text)) return;
|
|
146
190
|
if (reply.final && reply.consumed >= reply.raw.length) { voice.endSpeech(); return; }
|
|
147
191
|
}
|
|
@@ -152,7 +196,10 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
152
196
|
let reply = replies.get(message.id);
|
|
153
197
|
if (!reply) {
|
|
154
198
|
if (!primed || !readAloud.value || speechSuppressed || seen.has(message.id) || (!final && message.autonomous)) return;
|
|
155
|
-
reply = {
|
|
199
|
+
reply = {
|
|
200
|
+
id: message.id, outputId: message.outputId || message.id, streamId: message.streamId,
|
|
201
|
+
raw: "", consumed: 0, final: false, canonicalFinal: false, started: false
|
|
202
|
+
};
|
|
156
203
|
replies.set(message.id, reply);
|
|
157
204
|
speechQueue.push(reply);
|
|
158
205
|
}
|
|
@@ -162,25 +209,43 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
162
209
|
if (!text.startsWith(spokenPrefix) && !(final && text.trimEnd() === spokenPrefix.trimEnd())) {
|
|
163
210
|
// A model revision cannot unsay words. Retire that projection; never splice
|
|
164
211
|
// its old prefix onto a different answer or replay it as a second answer.
|
|
165
|
-
reply
|
|
212
|
+
cancelReply(reply);
|
|
166
213
|
if (speechQueue[0] === reply) voice.stopSpeaking();
|
|
167
214
|
return;
|
|
168
215
|
}
|
|
169
216
|
reply.consumed = Math.min(reply.consumed, text.length);
|
|
170
217
|
reply.raw = text;
|
|
218
|
+
reply.canonicalFinal ||= final;
|
|
171
219
|
reply.final = final || text.length >= 4000;
|
|
220
|
+
if (reply.canonicalFinal && reply.drained) emitReply(reply, "completed");
|
|
172
221
|
}
|
|
173
|
-
async function
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
222
|
+
async function enableSound() {
|
|
223
|
+
const revision = ++soundRevision;
|
|
224
|
+
soundPreparing.value = true;
|
|
225
|
+
try {
|
|
226
|
+
await voice.preparePlayback();
|
|
227
|
+
} catch {
|
|
228
|
+
// Requested sound stays on; its recovery control remains available.
|
|
229
|
+
} finally {
|
|
230
|
+
if (revision === soundRevision && !disposed) {
|
|
231
|
+
soundPreparing.value = false;
|
|
232
|
+
void drainSpeech();
|
|
233
|
+
}
|
|
234
|
+
}
|
|
178
235
|
}
|
|
179
236
|
async function toggleReadAloud() {
|
|
180
237
|
error.value = "";
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
238
|
+
readAloud.value = !readAloud.value;
|
|
239
|
+
observe(binding.onReadAloudChange, readAloud.value);
|
|
240
|
+
if (!readAloud.value) {
|
|
241
|
+
++soundRevision;
|
|
242
|
+
soundPreparing.value = false;
|
|
243
|
+
cancelSpeech("muted");
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
speechSuppressed = false;
|
|
247
|
+
speechInvitation = "";
|
|
248
|
+
await enableSound();
|
|
184
249
|
}
|
|
185
250
|
async function deliverTranscript() {
|
|
186
251
|
const pending = pendingTranscript.value;
|
|
@@ -199,6 +264,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
199
264
|
startupAbort?.abort();
|
|
200
265
|
startupAbort = null;
|
|
201
266
|
starting.value = false;
|
|
267
|
+
releaseEditCapturePause();
|
|
202
268
|
recording = null;
|
|
203
269
|
await voice.cancelListening();
|
|
204
270
|
}
|
|
@@ -209,11 +275,21 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
209
275
|
await cancelRecording();
|
|
210
276
|
}
|
|
211
277
|
function dismissTranscript() {
|
|
278
|
+
releaseEditCapturePause();
|
|
212
279
|
pendingTranscript.value = null;
|
|
213
280
|
heldReview.value = false;
|
|
214
281
|
holding.value = false;
|
|
215
282
|
error.value = "";
|
|
216
283
|
}
|
|
284
|
+
function releaseEditCapturePause() {
|
|
285
|
+
const pending = pendingTranscript.value;
|
|
286
|
+
const captureId = pending?.editPausedCaptureId;
|
|
287
|
+
if (!captureId) return;
|
|
288
|
+
delete pending.editPausedCaptureId;
|
|
289
|
+
if (recording?.messageId === captureId && !disposed && live.value && callMode.value === "hands-free") {
|
|
290
|
+
voice.setMicrophoneMuted(false);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
217
293
|
async function talk() {
|
|
218
294
|
error.value = "";
|
|
219
295
|
try {
|
|
@@ -224,7 +300,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
224
300
|
} catch (cause) { recording = null; error.value = cause.message; }
|
|
225
301
|
}
|
|
226
302
|
async function startRecording(reviewBeforeSend, continuous = false) {
|
|
227
|
-
if (disposed || starting.value || microphoneMuted.value || capturing.value ||
|
|
303
|
+
if (disposed || starting.value || microphoneMuted.value || capturing.value || committing ||
|
|
304
|
+
(pendingTranscript.value || sending.value) && !(continuous && live.value && (callMode.value === "hands-free" || pushHolding.value))) return;
|
|
228
305
|
error.value = "";
|
|
229
306
|
reviewBeforeSend ||= binding.defaults?.reviewBeforeSend === true;
|
|
230
307
|
heldReview.value = reviewBeforeSend;
|
|
@@ -232,20 +309,19 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
232
309
|
// Keep the starting target even if navigation changes during this utterance.
|
|
233
310
|
const next = { focus: { ...(binding.captureContext?.() || {}) }, messageId: crypto.randomUUID(), started: false, reviewBeforeSend, continuous };
|
|
234
311
|
recording = next;
|
|
235
|
-
if (!reviewBeforeSend && !live.value) {
|
|
236
|
-
readAloud.value = true;
|
|
237
|
-
void voice.preparePlayback().catch((cause) => { error.value = cause.message; });
|
|
238
|
-
}
|
|
239
312
|
try {
|
|
240
313
|
const started = await voice.startListening({ continuous });
|
|
241
314
|
if (recording === next) { next.started = started; if (!started) recording = null; }
|
|
242
315
|
} catch (cause) { if (recording === next) { recording = null; error.value = cause.message; } }
|
|
243
316
|
}
|
|
244
317
|
async function toggleMicrophoneMuted() {
|
|
318
|
+
// A person's explicit microphone choice supersedes an Edit-owned pause.
|
|
319
|
+
if (pendingTranscript.value) delete pendingTranscript.value.editPausedCaptureId;
|
|
245
320
|
voice.setMicrophoneMuted(!microphoneMuted.value);
|
|
321
|
+
if (!microphoneMuted.value && recording) delete recording.finishAfterPending;
|
|
246
322
|
clearTimeout(stopTimer);
|
|
247
323
|
if (microphoneMuted.value && starting.value) await cancelRecording();
|
|
248
|
-
else if (!microphoneMuted.value && live.value && callMode.value === "hands-free" && !capturing.value
|
|
324
|
+
else if (!microphoneMuted.value && live.value && callMode.value === "hands-free" && !capturing.value) await startRecording(false, true);
|
|
249
325
|
}
|
|
250
326
|
async function changeCallMode(mode) {
|
|
251
327
|
if (!["push-to-talk", "hands-free"].includes(mode) || mode === callMode.value || callModeBusy.value || committing) return;
|
|
@@ -258,13 +334,13 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
258
334
|
}
|
|
259
335
|
async function startPushToTalk(event) {
|
|
260
336
|
if (event?.repeat || event?.isPrimary === false || event?.button !== undefined && event.button !== 0 || pushHolding.value || disposed || starting.value
|
|
261
|
-
|| capturing.value && !voice.listening.value ||
|
|
337
|
+
|| capturing.value && !voice.listening.value || committing) return;
|
|
262
338
|
if (event?.pointerId !== undefined) event.currentTarget.setPointerCapture(event.pointerId);
|
|
263
339
|
pushInput = event?.pointerId ?? event?.key ?? null;
|
|
264
340
|
pushHolding.value = true;
|
|
265
341
|
callMode.value = "push-to-talk";
|
|
266
342
|
voice.setMicrophoneMuted(false);
|
|
267
|
-
|
|
343
|
+
if (recording) delete recording.finishAfterPending;
|
|
268
344
|
if (!live.value) await toggleLive();
|
|
269
345
|
if (!pushHolding.value || !live.value || disposed) return;
|
|
270
346
|
// A hold can take over an open hands-free microphone without throwing away
|
|
@@ -273,7 +349,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
273
349
|
recording.continuous = false;
|
|
274
350
|
return;
|
|
275
351
|
}
|
|
276
|
-
await startRecording(false);
|
|
352
|
+
await startRecording(false, Boolean(pendingTranscript.value || sending.value));
|
|
277
353
|
if (!recording) pushHolding.value = false;
|
|
278
354
|
}
|
|
279
355
|
async function finishPushToTalk(event) {
|
|
@@ -281,6 +357,11 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
281
357
|
pushHolding.value = false;
|
|
282
358
|
pushInput = null;
|
|
283
359
|
try {
|
|
360
|
+
if (voice.listening.value && (pendingTranscript.value || sending.value || committing)) {
|
|
361
|
+
if (recording) recording.finishAfterPending = true;
|
|
362
|
+
voice.setMicrophoneMuted(true);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
284
365
|
if (voice.listening.value) await voice.stopListening();
|
|
285
366
|
else await cancelRecording();
|
|
286
367
|
} catch (cause) { await cancelRecording(); error.value = cause.message; }
|
|
@@ -292,7 +373,15 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
292
373
|
await cancelRecording();
|
|
293
374
|
}
|
|
294
375
|
async function toggleHandsFree() {
|
|
295
|
-
if (disposed || pushHolding.value
|
|
376
|
+
if (disposed || pushHolding.value) return;
|
|
377
|
+
if (live.value && callMode.value === "hands-free" && voice.listening.value && (pendingTranscript.value || sending.value || committing)) {
|
|
378
|
+
// Pause must finish the newer recording after the existing admission,
|
|
379
|
+
// without replacing the one pending transcript or stopping output.
|
|
380
|
+
if (!microphoneMuted.value && recording) recording.finishAfterPending = true;
|
|
381
|
+
await toggleMicrophoneMuted();
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
if (committing) return;
|
|
296
385
|
if (starting.value) { await cancelRecording(); return; }
|
|
297
386
|
if (capturing.value && !voice.listening.value) return;
|
|
298
387
|
if (live.value && callMode.value === "hands-free") {
|
|
@@ -301,10 +390,16 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
301
390
|
// final sound is included; resuming starts a fresh recording.
|
|
302
391
|
await voice.stopListening();
|
|
303
392
|
voice.setMicrophoneMuted(true);
|
|
304
|
-
} else await toggleMicrophoneMuted();
|
|
393
|
+
} else if (microphoneMuted.value) await toggleMicrophoneMuted();
|
|
394
|
+
else await startRecording(false, true);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (callMode.value === "hands-free" && (pendingTranscript.value || sending.value)) {
|
|
398
|
+
voice.setMicrophoneMuted(false);
|
|
399
|
+
await toggleLive();
|
|
305
400
|
return;
|
|
306
401
|
}
|
|
307
|
-
if (callModeBusy.value) return;
|
|
402
|
+
if (pendingTranscript.value || sending.value || callModeBusy.value) return;
|
|
308
403
|
voice.setMicrophoneMuted(false);
|
|
309
404
|
await changeCallMode("hands-free");
|
|
310
405
|
if (!live.value) await toggleLive();
|
|
@@ -318,27 +413,23 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
318
413
|
committing = null;
|
|
319
414
|
clearTimeout(stopTimer);
|
|
320
415
|
await cancelRecording();
|
|
321
|
-
stopSpeech();
|
|
322
|
-
readAloud.value = false;
|
|
323
|
-
void voice.close().catch(cause => { if (!live.value) error.value = cause.message; });
|
|
324
416
|
return;
|
|
325
417
|
}
|
|
326
|
-
if (disposed || microphoneMuted.value || capturing.value ||
|
|
418
|
+
if (disposed || microphoneMuted.value || capturing.value || committing ||
|
|
419
|
+
(pendingTranscript.value || sending.value) && callMode.value !== "hands-free" && !pushHolding.value) return;
|
|
327
420
|
const controller = new AbortController();
|
|
328
421
|
startupAbort = controller;
|
|
329
422
|
starting.value = true;
|
|
330
423
|
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(10000)]);
|
|
331
424
|
try {
|
|
332
|
-
// Unlock
|
|
333
|
-
|
|
334
|
-
await voice.preparePlayback();
|
|
425
|
+
// Unlock requested sound in the gesture without delaying microphone startup.
|
|
426
|
+
if (readAloud.value) void enableSound();
|
|
335
427
|
signal.throwIfAborted();
|
|
336
428
|
await voice.connect();
|
|
337
429
|
signal.throwIfAborted();
|
|
338
430
|
if (disposed) return;
|
|
339
431
|
starting.value = false;
|
|
340
432
|
live.value = true;
|
|
341
|
-
readAloud.value = true;
|
|
342
433
|
if (callMode.value === "hands-free") {
|
|
343
434
|
await startRecording(false, true);
|
|
344
435
|
if (startupAbort === controller && !recording?.started) live.value = false;
|
|
@@ -352,12 +443,6 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
352
443
|
if (startupAbort === controller) { startupAbort = null; starting.value = false; }
|
|
353
444
|
}
|
|
354
445
|
}
|
|
355
|
-
function retireAnswer(smooth = false) {
|
|
356
|
-
for (const reply of speechQueue) reply.cancelled = true;
|
|
357
|
-
speechQueue.length = 0;
|
|
358
|
-
if (smooth) voice.finishCurrentPhrase();
|
|
359
|
-
else voice.stopSpeaking();
|
|
360
|
-
}
|
|
361
446
|
function currentSpokenText() {
|
|
362
447
|
const reply = speechQueue[0];
|
|
363
448
|
return reply ? reply.raw.slice(0, reply.consumed).slice(-2000) : "";
|
|
@@ -367,7 +452,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
367
452
|
}
|
|
368
453
|
function decideTurn() {
|
|
369
454
|
const candidate = voice.endpoint.value;
|
|
370
|
-
if (!live.value || callMode.value !== "hands-free" || !voice.listening.value || microphoneMuted.value || pendingTranscript.value || sending.value || !candidate) return;
|
|
455
|
+
if (takingTranscript.value || !live.value || callMode.value !== "hands-free" || !voice.listening.value || microphoneMuted.value || pendingTranscript.value || sending.value || !candidate) return;
|
|
371
456
|
const key = `${candidate.turnId}:${candidate.revision}`;
|
|
372
457
|
if (decidedCandidate === key) return;
|
|
373
458
|
decidedCandidate = key;
|
|
@@ -393,9 +478,9 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
393
478
|
}
|
|
394
479
|
committing = { ...captured, turnId: candidate.turnId, revision: candidate.revision };
|
|
395
480
|
if (!voice.finishUtterance(candidate)) { committing = null; return; }
|
|
481
|
+
publishTranscript();
|
|
396
482
|
// The daemon rechecks the endpoint revision before returning final text.
|
|
397
|
-
//
|
|
398
|
-
retireAnswer(true);
|
|
483
|
+
// Ordinary steering leaves the current answer and queued audio intact.
|
|
399
484
|
} catch (cause) {
|
|
400
485
|
if (live.value && candidate === voice.endpoint.value) {
|
|
401
486
|
live.value = false;
|
|
@@ -404,7 +489,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
404
489
|
}
|
|
405
490
|
}
|
|
406
491
|
}
|
|
407
|
-
watch([voice.endpoint, pendingTranscript, sending], decideTurn);
|
|
492
|
+
watch([voice.endpoint, pendingTranscript, sending, takingTranscript], decideTurn);
|
|
408
493
|
watch(voice.completedUtterance, (utterance) => {
|
|
409
494
|
if (!utterance || !live.value || !committing || committing.turnId !== utterance.turnId || committing.revision !== utterance.revision) return;
|
|
410
495
|
pendingTranscript.value = { ...committing, text: utterance.text };
|
|
@@ -415,7 +500,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
415
500
|
// A discard can lose a race with resumed speech. Keep this utterance's
|
|
416
501
|
// identity and destination until the daemon acknowledges the boundary.
|
|
417
502
|
if (reset?.turnId !== voice.activeListenTurnId.value || !live.value || !recording) return;
|
|
418
|
-
recording = { focus: { ...(binding.captureContext?.() || {}) }, messageId: crypto.randomUUID(), started: true, continuous: true
|
|
503
|
+
recording = { focus: { ...(binding.captureContext?.() || {}) }, messageId: crypto.randomUUID(), started: true, continuous: true,
|
|
504
|
+
...(recording.finishAfterPending ? { finishAfterPending: true } : {}) };
|
|
419
505
|
});
|
|
420
506
|
watch(voice.partialTranscript, (text, previous) => {
|
|
421
507
|
clearTimeout(stopTimer);
|
|
@@ -426,7 +512,16 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
426
512
|
if (live.value && !microphoneMuted.value && voice.partialTranscript.value === text) stopSpeech();
|
|
427
513
|
}, 120);
|
|
428
514
|
});
|
|
429
|
-
watch([pendingTranscript, sending], () => {
|
|
515
|
+
watch([pendingTranscript, sending, voice.utteranceReset], () => {
|
|
516
|
+
if (!disposed && live.value && !pushHolding.value && !changingCallMode.value &&
|
|
517
|
+
!pendingTranscript.value && !sending.value && !committing && microphoneMuted.value &&
|
|
518
|
+
recording?.finishAfterPending && voice.listening.value) {
|
|
519
|
+
// Retire intent before the asynchronous PCM drain so a receipt/reset
|
|
520
|
+
// cannot finalize this capture twice. Its original final watcher sends B.
|
|
521
|
+
delete recording.finishAfterPending;
|
|
522
|
+
void voice.stopListening().catch(cause => { error.value = cause.message; });
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
430
525
|
if (live.value && callMode.value === "hands-free" && !changingCallMode.value && !microphoneMuted.value && !capturing.value && !pendingTranscript.value && !sending.value && !voice.error.value) void startRecording(false, true);
|
|
431
526
|
});
|
|
432
527
|
async function startHeldRecording() { await startRecording(true); }
|
|
@@ -477,10 +572,12 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
477
572
|
text: binding.state.streamingReply?.text }), ({ messages = [], reply }) => {
|
|
478
573
|
if (!binding.id) return;
|
|
479
574
|
// Reconcile an uncertain send by identity, never by equal transcript text.
|
|
480
|
-
if (pendingTranscript.value && messages.some((message) => message.role === "user" && message.id === pendingTranscript.value.messageId)) {
|
|
575
|
+
if (pendingTranscript.value && messages.some((message) => message.role === "user" && message.id === pendingTranscript.value.messageId && message.receipt !== false)) {
|
|
481
576
|
dismissTranscript();
|
|
482
577
|
}
|
|
483
578
|
for (const message of messages) {
|
|
579
|
+
// Unadmitted local records do not acknowledge a request or lift speech silence.
|
|
580
|
+
if (message.role === "user" && message.receipt === false) continue;
|
|
484
581
|
if (seen.has(message.id)) continue;
|
|
485
582
|
if (message.role === "user" && message.id === speechInvitation) {
|
|
486
583
|
speechSuppressed = false;
|
|
@@ -491,8 +588,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
491
588
|
}
|
|
492
589
|
if (lastStreamingId && lastStreamingId !== reply?.id && !messages.some(message => message.id === lastStreamingId)) {
|
|
493
590
|
const retired = replies.get(lastStreamingId);
|
|
494
|
-
if (retired && !retired.
|
|
495
|
-
retired
|
|
591
|
+
if (retired && !retired.canonicalFinal && !retired.cancelled) {
|
|
592
|
+
cancelReply(retired);
|
|
496
593
|
if (speechQueue[0] === retired) voice.finishCurrentPhrase();
|
|
497
594
|
}
|
|
498
595
|
}
|
|
@@ -502,22 +599,79 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
502
599
|
if (!readAloud.value || !primed || reply.autonomous) seen.add(reply.id);
|
|
503
600
|
}
|
|
504
601
|
// Keep only the visible history plus active playback receipts in memory.
|
|
505
|
-
const retained = new Set([...messages.map((message) => message.id), reply?.id, ...speechQueue.map((item) => item.id)
|
|
602
|
+
const retained = new Set([...messages.map((message) => message.id), reply?.id, ...speechQueue.map((item) => item.id),
|
|
603
|
+
...[...replies.values()].filter(item => !item.terminal).map(item => item.id)]);
|
|
506
604
|
for (const id of seen) if (!retained.has(id)) seen.delete(id);
|
|
507
605
|
for (const id of replies.keys()) if (!retained.has(id)) replies.delete(id);
|
|
508
606
|
primed = true;
|
|
509
607
|
void drainSpeech();
|
|
510
608
|
}, { immediate: true });
|
|
511
|
-
watch(voice.error, (message) => { if (message)
|
|
609
|
+
watch(voice.error, (message) => { if (message) cancelSpeech("playback-error", "failed"); });
|
|
512
610
|
watch(() => error.value || voice.error.value, (message) => { if (message) binding.onError?.(message); });
|
|
513
611
|
watch([voice.activeSpeechTurnId, voice.canAppendSpeech], () => { void drainSpeech(); });
|
|
514
|
-
|
|
612
|
+
function currentTranscript() {
|
|
515
613
|
const pending = pendingTranscript.value;
|
|
516
|
-
const
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
}
|
|
614
|
+
const currentWords = recording && ["listening", "transcribing"].includes(voice.captureState.value)
|
|
615
|
+
&& voice.partialTranscript.value && recording.messageId !== pending?.messageId;
|
|
616
|
+
return currentWords ? { id: recording.messageId, text: voice.partialTranscript.value }
|
|
617
|
+
: pending ? { id: pending.messageId, text: pending.text } : null;
|
|
618
|
+
}
|
|
619
|
+
function canTakeTranscript(messageId) {
|
|
620
|
+
return !disposed && !starting.value && !sending.value && !committing && !takingTranscript.value
|
|
621
|
+
&& (pendingTranscript.value?.messageId === messageId || currentTranscript()?.id === messageId)
|
|
622
|
+
&& !binding.state.messages?.some(message => message.role === "user" && message.id === messageId && message.receipt !== false);
|
|
623
|
+
}
|
|
624
|
+
async function beginTranscriptEdit(messageId) {
|
|
625
|
+
if (!canTakeTranscript(messageId)) return false;
|
|
626
|
+
const pending = pendingTranscript.value;
|
|
627
|
+
if (pending && pending.messageId !== messageId) return false;
|
|
628
|
+
const selected = pending ? { id: pending.messageId, text: pending.text } : currentTranscript();
|
|
629
|
+
const capture = recording;
|
|
630
|
+
takingTranscript.value = true;
|
|
631
|
+
try {
|
|
632
|
+
// Retain the selected words and destination before cancelling capture.
|
|
633
|
+
// The existing pending-clear watcher resumes hands-free after resolution.
|
|
634
|
+
pendingTranscript.value = {
|
|
635
|
+
...(pending?.messageId === messageId ? pending : capture),
|
|
636
|
+
messageId, text: selected.text, reviewBeforeSend: true, editing: true
|
|
637
|
+
};
|
|
638
|
+
if (capture) {
|
|
639
|
+
if (capture.messageId !== messageId && live.value && callMode.value === "hands-free") {
|
|
640
|
+
// Newer continuous words retain their recording identity and endpoint.
|
|
641
|
+
if (!microphoneMuted.value) {
|
|
642
|
+
pendingTranscript.value.editPausedCaptureId = capture.messageId;
|
|
643
|
+
voice.setMicrophoneMuted(true);
|
|
644
|
+
}
|
|
645
|
+
} else if (pushHolding.value) await cancelPushToTalk();
|
|
646
|
+
else await cancelRecording();
|
|
647
|
+
}
|
|
648
|
+
return true;
|
|
649
|
+
} finally { takingTranscript.value = false; }
|
|
650
|
+
}
|
|
651
|
+
async function takeTranscript(messageId, transfer) {
|
|
652
|
+
if (!canTakeTranscript(messageId)) return false;
|
|
653
|
+
const pending = pendingTranscript.value;
|
|
654
|
+
const selected = pending?.messageId === messageId ? { id: messageId, text: pending.text } : currentTranscript();
|
|
655
|
+
takingTranscript.value = true;
|
|
656
|
+
try {
|
|
657
|
+
// The host rechecks its draft and transfers synchronously before cancellation.
|
|
658
|
+
if (transfer && transfer(selected.text) !== true) return false;
|
|
659
|
+
if (pendingTranscript.value?.messageId === messageId) dismissTranscript();
|
|
660
|
+
else if (recording?.messageId === messageId) {
|
|
661
|
+
if (pushHolding.value) await cancelPushToTalk();
|
|
662
|
+
else await cancelRecording();
|
|
663
|
+
if (!disposed && live.value && callMode.value === "hands-free" && !microphoneMuted.value) {
|
|
664
|
+
await startRecording(false, true);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
return true;
|
|
668
|
+
} finally { takingTranscript.value = false; }
|
|
669
|
+
}
|
|
670
|
+
function publishTranscript() {
|
|
671
|
+
const transcript = currentTranscript();
|
|
672
|
+
binding.onTranscript?.(transcript, { canTake: Boolean(transcript && canTakeTranscript(transcript.id)) });
|
|
673
|
+
}
|
|
674
|
+
watch([pendingTranscript, voice.captureState, voice.partialTranscript, voice.endpoint, sending, starting, takingTranscript], publishTranscript, { immediate: true });
|
|
521
675
|
watch(avatarVisual, (visual) => binding.onVisual?.(visual), { immediate: true });
|
|
522
676
|
const page = globalThis.window || globalThis;
|
|
523
677
|
const cancelStartup = () => { if (starting.value) void discardRecording(); else void cancelPushToTalk(); };
|
|
@@ -527,6 +681,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
527
681
|
page.removeEventListener?.("pagehide", cancelStartup);
|
|
528
682
|
page.removeEventListener?.("blur", cancelPushToTalk);
|
|
529
683
|
startupAbort?.abort(); startupAbort = null;
|
|
684
|
+
stopSpeech();
|
|
530
685
|
binding.onTranscript?.(null); binding.onVisual?.(null); disposed = true; live.value = false; clearTimeout(stopTimer); recording = null; speechQueue.length = 0;
|
|
531
686
|
});
|
|
532
687
|
|
|
@@ -540,18 +695,21 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
540
695
|
stopSpeech();
|
|
541
696
|
await voice.close();
|
|
542
697
|
}
|
|
543
|
-
function editTranscript(text) {
|
|
544
|
-
if (pendingTranscript.value && !sending.value) pendingTranscript.value = { ...pendingTranscript.value, text: String(text) };
|
|
698
|
+
function editTranscript(text, messageId = pendingTranscript.value?.messageId) {
|
|
699
|
+
if (pendingTranscript.value && pendingTranscript.value.messageId === messageId && !sending.value) pendingTranscript.value = { ...pendingTranscript.value, text: String(text) };
|
|
545
700
|
}
|
|
701
|
+
const composerBlocked = computed(() => starting.value ||
|
|
702
|
+
(!live.value || callMode.value !== "hands-free") &&
|
|
703
|
+
(sending.value || Boolean(pendingTranscript.value) || capturing.value));
|
|
546
704
|
const hasUnsentSpeech = computed(() => capturing.value || starting.value || sending.value || Boolean(pendingTranscript.value));
|
|
547
705
|
return {
|
|
548
706
|
oneOffTalkMode, targetLabel, voice, readAloud, error, pendingTranscript, heldReview, holding, sending,
|
|
549
707
|
live, starting, callMode, changingCallMode, pushHolding, microphoneMuted, capturing,
|
|
550
708
|
speechActive, talkLabel, liveLabel, callConnection, callStatus, callDetail, callModeBusy,
|
|
551
709
|
microphoneStatus, avatarVisual, callAudioLevel, status, heldTranscript, voiceWords, voiceAnswer,
|
|
552
|
-
stopSpeech, inviteSpeech, toggleReadAloud, deliverTranscript, cancelRecording,
|
|
710
|
+
stopSpeech, inviteSpeech, toggleReadAloud, enableSound, soundPreparing, deliverTranscript, cancelRecording,
|
|
553
711
|
discardRecording, dismissTranscript, talk, toggleMicrophoneMuted, changeCallMode,
|
|
554
712
|
startPushToTalk, finishPushToTalk, cancelPushToTalk, toggleLive, toggleHandsFree, startHeldRecording,
|
|
555
|
-
finishHeldRecording, discardHeldRecording, close, editTranscript, hasUnsentSpeech
|
|
713
|
+
finishHeldRecording, discardHeldRecording, close, editTranscript, canTakeTranscript, beginTranscriptEdit, takeTranscript, hasUnsentSpeech, composerBlocked
|
|
556
714
|
};
|
|
557
715
|
}
|