@jskit-ai/assistant-voice 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/examples/conversation/App.vue +2 -1
- package/examples/conversation/package.json +8 -8
- package/package.json +2 -2
- package/src/client/ConversationDialog.vue +23 -12
- package/src/client/VoiceConversation.vue +19 -4
- package/src/client/VoiceConversationControls.vue +83 -35
- package/src/client/VoiceConversationHost.vue +20 -25
- package/src/client/VoiceConversationSettings.vue +52 -0
- package/src/client/index.js +1 -0
- package/src/client/voiceConversation.js +274 -28
package/README.md
CHANGED
|
@@ -83,9 +83,45 @@ The host's `avatar` slot receives state, mouthLevel, mouthPose, avatar and bindi
|
|
|
83
83
|
Opening a target does not start capture. Tap Talk for hands-free, or hold it for
|
|
84
84
|
push-to-talk and release to send; connection setup is automatic.
|
|
85
85
|
Pause finishes the current utterance, flushing the audio tail before muting.
|
|
86
|
-
Completed words follow the captured review policy
|
|
86
|
+
Completed words follow the captured review policy. Pending delivery does not disable
|
|
87
|
+
the microphone: tap can resume the existing continuous capture or start another
|
|
88
|
+
one while preserving the prior pending transcript. A busy Pause or hold release
|
|
89
|
+
mutes newer capture and finishes it through the original owner after that prior
|
|
90
|
+
admission settles. Resuming first keeps the capture open. No pending words are
|
|
91
|
+
replaced or appended. The icon reflects actual unmuted capture, not readiness.
|
|
87
92
|
Automatic sends keep the call layout fixed. A failed send exposes the retained
|
|
88
|
-
transcript for review and retry with the same message identity.
|
|
93
|
+
transcript for review and retry with the same message identity. An application
|
|
94
|
+
can call `beginTranscriptEdit(messageId)` for its exact unsent preview; this
|
|
95
|
+
retains the existing pending transcript and capture destination before cancelling
|
|
96
|
+
only capture. `editTranscript(text, messageId)` updates that pending identity and
|
|
97
|
+
`deliverTranscript()` keeps the original send/retry owner. Hands-free resumes
|
|
98
|
+
through the existing pending-clear watcher after send/discard, not during editing.
|
|
99
|
+
An adapter's `previewMessage.actions` may project `editing`, `update(id, text)`,
|
|
100
|
+
`send(id)`, `canSend` and `sending` alongside its original ×/edit actions. The
|
|
101
|
+
standard transcript uses the same review textarea inside that preview bubble,
|
|
102
|
+
including an empty editor, and reuses its existing action row for pending Send.
|
|
103
|
+
A matching canonical user with `receipt: false` is still unadmitted: it does not
|
|
104
|
+
clear pending words, forbid Edit or acknowledge a speech invitation. Confirmed
|
|
105
|
+
readback (an absent flag or `receipt: true`) retains the original identity-based
|
|
106
|
+
acknowledgement. For a known failed local delivery, a host can cancel that exact
|
|
107
|
+
entry through the original delivery owner only upon explicit Edit, then author
|
|
108
|
+
fresh edited text and intent on Send. Plain Retry retains its original payload;
|
|
109
|
+
uncertain, accepted and sending entries remain fenced.
|
|
110
|
+
When continuous recognition has newer words while an earlier pending send fails,
|
|
111
|
+
both identities remain in their original pending/recording owners. Hosts can
|
|
112
|
+
project these two existing snapshots through `previewMessages`, deduplicated by
|
|
113
|
+
ID, while retaining the singular `previewMessage` contract. `canTakeTranscript(id)`
|
|
114
|
+
checks either exact existing identity with the original admission guards. The
|
|
115
|
+
newer capture is discardable, but cannot replace the occupied pending editor.
|
|
116
|
+
Editing the earlier transcript pauses the newer recording through the original
|
|
117
|
+
mute owner, retaining its words, ID, focus and endpoint. Transient
|
|
118
|
+
`editPausedCaptureId` on the pending transcript owns only this pause; an explicit
|
|
119
|
+
microphone choice supersedes it. Original pending resolution releases only the
|
|
120
|
+
matching Edit-owned pause, and the existing endpoint watcher handles newer words.
|
|
121
|
+
No additional transcript queue, playback owner or capture process is created.
|
|
122
|
+
The app must retain current-target and uncertain-admission guards. Controls use
|
|
123
|
+
`reviewInTranscript` only while that preview is projected, suppressing a duplicate
|
|
124
|
+
inline review; standalone voice retains its original review fallback. Captions,
|
|
89
125
|
review, explicit microphone/sound controls and bounded reconnection share one
|
|
90
126
|
session. Opening the same ID reveals it. Switching targets releases the previous
|
|
91
127
|
audio first; unfinished words require completion or explicit discard. A pending
|
|
@@ -10,6 +10,7 @@ const closed = ref([]);
|
|
|
10
10
|
const views = new Map();
|
|
11
11
|
const error = ref("");
|
|
12
12
|
const avatarSize = ref("compact");
|
|
13
|
+
const voicePresentation = ref("dialog");
|
|
13
14
|
const acquireConversation = useAssistantConversationFactory();
|
|
14
15
|
const controller = createVoiceConversationController({ connectSpeech: binding => binding.socketUrl });
|
|
15
16
|
async function openText(id) {
|
|
@@ -91,7 +92,7 @@ onScopeDispose(() => { void controller.dispose(); });
|
|
|
91
92
|
</section>
|
|
92
93
|
</div>
|
|
93
94
|
</main>
|
|
94
|
-
<VoiceConversationHost v-model:avatar-size="avatarSize" :controller="controller" />
|
|
95
|
+
<VoiceConversationHost v-model:avatar-size="avatarSize" :controller="controller" :presentation="voicePresentation" />
|
|
95
96
|
<ShellErrorHost />
|
|
96
97
|
</v-app>
|
|
97
98
|
</template>
|
|
@@ -9,14 +9,14 @@
|
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@fastify/websocket": "^11.2.0",
|
|
12
|
-
"@jskit-ai/assistant-core": "0.1.
|
|
13
|
-
"@jskit-ai/assistant-runtime": "0.1.
|
|
14
|
-
"@jskit-ai/assistant-voice": "0.1.
|
|
15
|
-
"@jskit-ai/connectors-catalog": "0.1.
|
|
16
|
-
"@jskit-ai/http-runtime": "0.1.
|
|
17
|
-
"@jskit-ai/kernel": "0.1.
|
|
18
|
-
"@jskit-ai/realtime": "0.1.
|
|
19
|
-
"@jskit-ai/shell-web": "0.1.
|
|
12
|
+
"@jskit-ai/assistant-core": "0.1.206",
|
|
13
|
+
"@jskit-ai/assistant-runtime": "0.1.199",
|
|
14
|
+
"@jskit-ai/assistant-voice": "0.1.6",
|
|
15
|
+
"@jskit-ai/connectors-catalog": "0.1.45",
|
|
16
|
+
"@jskit-ai/http-runtime": "0.1.226",
|
|
17
|
+
"@jskit-ai/kernel": "0.1.228",
|
|
18
|
+
"@jskit-ai/realtime": "0.1.225",
|
|
19
|
+
"@jskit-ai/shell-web": "0.1.232",
|
|
20
20
|
"@mdi/js": "^7.4.47",
|
|
21
21
|
"@tanstack/vue-query": "^5.90.5",
|
|
22
22
|
"fastify": "^5.8.0",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jskit-ai/assistant-voice",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Duplex voice conversations with application-owned targets, authorization and speech services.",
|
|
6
6
|
"scripts": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"examples"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@jskit-ai/assistant-core": "0.1.
|
|
21
|
+
"@jskit-ai/assistant-core": "0.1.206",
|
|
22
22
|
"@mdi/js": "^7.4.47",
|
|
23
23
|
"ws": "^8.20.1"
|
|
24
24
|
},
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
<script setup>
|
|
2
2
|
import { mdiClose, mdiMessageTextOutline, mdiMicrophone, mdiMinus } from "@mdi/js";
|
|
3
3
|
import { useDisplay } from "vuetify";
|
|
4
|
+
import { VCard, VDialog } from "vuetify/components";
|
|
5
|
+
import { computed } from "vue";
|
|
4
6
|
|
|
5
7
|
const { xs } = useDisplay();
|
|
6
8
|
|
|
@@ -11,7 +13,16 @@ const props = defineProps({
|
|
|
11
13
|
mode: { type: String, default: "talk" },
|
|
12
14
|
showModes: Boolean,
|
|
13
15
|
minimizable: Boolean,
|
|
14
|
-
closeLabel: { type: String, default: "Close conversation" }
|
|
16
|
+
closeLabel: { type: String, default: "Close conversation" },
|
|
17
|
+
presentation: { type: String, default: "dialog", validator: value => ["dialog", "inline"].includes(value) }
|
|
18
|
+
});
|
|
19
|
+
const dialogProps = computed(() => props.presentation === "inline" ? {} : {
|
|
20
|
+
activator: props.activator, openOnClick: false, modelValue: props.modelValue,
|
|
21
|
+
transition: false, fullscreen: xs.value, width: xs.value ? "100%" : 620,
|
|
22
|
+
maxWidth: xs.value ? "100%" : "calc(100% - 32px)", height: xs.value ? "100%" : 860,
|
|
23
|
+
maxHeight: xs.value ? "100%" : "calc(100% - 32px)",
|
|
24
|
+
contentProps: { style: { margin: xs.value ? 0 : "16px" } },
|
|
25
|
+
"aria-hidden": !props.modelValue
|
|
15
26
|
});
|
|
16
27
|
const emit = defineEmits(["update:modelValue", "update:mode", "minimize"]);
|
|
17
28
|
let dismissedFromActivator = false;
|
|
@@ -29,18 +40,16 @@ function updateOpen(value) {
|
|
|
29
40
|
</script>
|
|
30
41
|
|
|
31
42
|
<template>
|
|
32
|
-
<
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
:height="xs ? '100%' : 860" :max-height="xs ? '100%' : 'calc(100% - 32px)'"
|
|
36
|
-
:content-props="{ style: { margin: xs ? 0 : '16px' } }" :aria-label="`${title} conversation`" :aria-hidden="!modelValue"
|
|
43
|
+
<component
|
|
44
|
+
:is="presentation === 'inline' ? 'section' : VDialog" v-bind="dialogProps"
|
|
45
|
+
class="conversation-dialog" :class="{ 'conversation-dialog--inline': presentation === 'inline' }" :aria-label="`${title} conversation`"
|
|
37
46
|
@click:outside="outsideClick" @update:model-value="updateOpen"
|
|
38
47
|
>
|
|
39
|
-
<
|
|
40
|
-
<v-card-title class="conversation-dialog__title text-title-large">
|
|
48
|
+
<component :is="presentation === 'inline' ? 'div' : VCard" class="conversation-dialog__surface" :rounded="presentation === 'inline' ? undefined : xs ? 0 : 'lg'">
|
|
49
|
+
<v-card-title v-if="presentation !== 'inline'" class="conversation-dialog__title text-title-large">
|
|
41
50
|
<span :title="title">{{ title }}</span>
|
|
42
51
|
<slot name="header-actions" />
|
|
43
|
-
<v-btn v-if="minimizable" :icon="mdiMinus" variant="text" aria-label="Minimize conversation" title="Minimize conversation" @click="$emit('minimize')" />
|
|
52
|
+
<v-btn v-if="minimizable" :icon="mdiMinus" variant="text" aria-label="Minimize conversation" title="Minimize conversation" @click="$emit('minimize', $event)" />
|
|
44
53
|
<v-btn :icon="mdiClose" variant="text" :aria-label="closeLabel" :title="closeLabel" @click="$emit('update:modelValue', false)" />
|
|
45
54
|
</v-card-title>
|
|
46
55
|
<v-tabs v-if="showModes" :model-value="mode" class="conversation-dialog__tabs" grow aria-label="Conversation mode" @update:model-value="$emit('update:mode', $event)">
|
|
@@ -48,8 +57,8 @@ function updateOpen(value) {
|
|
|
48
57
|
<v-tab value="text" :prepend-icon="mdiMessageTextOutline">Text</v-tab>
|
|
49
58
|
</v-tabs>
|
|
50
59
|
<div class="conversation-dialog__body"><slot /></div>
|
|
51
|
-
</
|
|
52
|
-
</
|
|
60
|
+
</component>
|
|
61
|
+
</component>
|
|
53
62
|
</template>
|
|
54
63
|
|
|
55
64
|
<style scoped>
|
|
@@ -57,11 +66,13 @@ function updateOpen(value) {
|
|
|
57
66
|
.conversation-dialog :deep(.v-overlay__scrim) { transition: none; opacity: var(--v-overlay-opacity); }
|
|
58
67
|
.conversation-dialog:not(.v-overlay--active) :deep(.v-overlay__scrim) { display: none; }
|
|
59
68
|
.conversation-dialog__surface { display: flex; flex-direction: column; height: 100%; min-height: 0; overflow: hidden; }
|
|
69
|
+
.conversation-dialog--inline { display: flex; flex-direction: column; min-height: 0; min-width: 0; }
|
|
70
|
+
.conversation-dialog--inline .conversation-dialog__surface { height: auto; }
|
|
60
71
|
.conversation-dialog__title { display: flex; align-items: center; flex: 0 0 64px; height: 64px; gap: 8px; padding: 8px 16px; }
|
|
61
72
|
.conversation-dialog__title > span { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
62
73
|
.conversation-dialog__tabs { flex: 0 0 48px; height: 48px; }
|
|
63
74
|
.conversation-dialog__body { display: flex; flex-direction: column; flex: 1; min-height: 0; overflow: hidden; }
|
|
64
75
|
@media (max-width: 599px) {
|
|
65
|
-
.conversation-dialog__surface { padding-top: env(safe-area-inset-top); padding-bottom: env(safe-area-inset-bottom); }
|
|
76
|
+
.conversation-dialog:not(.conversation-dialog--inline) .conversation-dialog__surface { padding-top: env(safe-area-inset-top); padding-bottom: env(safe-area-inset-bottom); }
|
|
66
77
|
}
|
|
67
78
|
</style>
|
|
@@ -3,6 +3,7 @@ import { computed, ref, watch } from "vue";
|
|
|
3
3
|
import { AssistantConversationElement } from "@jskit-ai/assistant-core/client/conversation";
|
|
4
4
|
import VoiceAvatar from "./VoiceAvatar.vue";
|
|
5
5
|
import VoiceConversationControls from "./VoiceConversationControls.vue";
|
|
6
|
+
import { mdiMicrophone, mdiVolumeHigh } from "@mdi/js";
|
|
6
7
|
|
|
7
8
|
const props = defineProps({
|
|
8
9
|
session: { type: Object, required: true }, disabled: { type: Boolean, default: false },
|
|
@@ -15,7 +16,7 @@ const { targetLabel, voice, microphoneMuted, heldTranscript, pendingTranscript,
|
|
|
15
16
|
avatarVisual, callAudioLevel, voiceWords, voiceAnswer } = props.session;
|
|
16
17
|
const presentationAdapter = computed(() => props.adapter && ({
|
|
17
18
|
...props.adapter,
|
|
18
|
-
composer: { ...props.adapter.composer, canSend: props.adapter.composer?.canSend && !props.session.
|
|
19
|
+
composer: { ...props.adapter.composer, canSend: props.adapter.composer?.canSend && !props.session.composerBlocked.value }
|
|
19
20
|
}));
|
|
20
21
|
const toolsTarget = ref(null);
|
|
21
22
|
const voiceWordsTarget = ref(null);
|
|
@@ -38,10 +39,21 @@ for (const [text, target, following] of [[voiceWords, voiceWordsTarget, followWo
|
|
|
38
39
|
<VoiceAvatar v-bind="avatarVisual" />
|
|
39
40
|
</slot>
|
|
40
41
|
</template>
|
|
41
|
-
<template #
|
|
42
|
+
<template v-if="showAvatar" #avatar-tools><div ref="toolsTarget" class="assistant-voice-conversation__tools" /></template>
|
|
43
|
+
<template #avatar-control="scope">
|
|
44
|
+
<slot name="avatar-control" v-bind="scope">
|
|
45
|
+
<span v-if="scope.size === 'hidden' && voice.listening.value && !microphoneMuted" role="img" aria-label="Listening" title="Listening">
|
|
46
|
+
<v-icon :icon="mdiMicrophone" size="20" aria-hidden="true" />
|
|
47
|
+
</span>
|
|
48
|
+
<span v-if="scope.size === 'hidden' && voice.speaking.value" role="img" aria-label="Speaking" title="Speaking">
|
|
49
|
+
<v-icon :icon="mdiVolumeHigh" size="20" aria-hidden="true" />
|
|
50
|
+
</span>
|
|
51
|
+
</slot>
|
|
52
|
+
</template>
|
|
42
53
|
<template #composer-feedback>
|
|
43
|
-
<VoiceConversationControls :session="session" :disabled="disabled" compact :tools-target="toolsTarget">
|
|
54
|
+
<VoiceConversationControls :session="session" :disabled="disabled" compact :icon-only="showAvatar" :tools-target="toolsTarget" :review-in-transcript="Boolean(presentationAdapter?.conversation?.previewMessage)">
|
|
44
55
|
<template #work-control><slot name="work-control" /></template>
|
|
56
|
+
<template #settings-control><slot name="settings-control" /></template>
|
|
45
57
|
</VoiceConversationControls>
|
|
46
58
|
</template>
|
|
47
59
|
</AssistantConversationElement>
|
|
@@ -66,7 +78,10 @@ for (const [text, target, following] of [[voiceWords, voiceWordsTarget, followWo
|
|
|
66
78
|
<p class="assistant-voice__caption-text text-body-medium">{{ voiceAnswer }}</p>
|
|
67
79
|
</section>
|
|
68
80
|
</div>
|
|
69
|
-
<VoiceConversationControls :session="session" :disabled="disabled"
|
|
81
|
+
<VoiceConversationControls :session="session" :disabled="disabled">
|
|
82
|
+
<template #work-control><slot name="work-control" /></template>
|
|
83
|
+
<template #settings-control><slot name="settings-control" /></template>
|
|
84
|
+
</VoiceConversationControls>
|
|
70
85
|
</section>
|
|
71
86
|
</fieldset>
|
|
72
87
|
</template>
|
|
@@ -6,8 +6,12 @@ import { useVoiceLauncher } from "./voiceLauncher.js";
|
|
|
6
6
|
const props = defineProps({
|
|
7
7
|
session: { type: Object, required: true },
|
|
8
8
|
disabled: { type: Boolean, default: false },
|
|
9
|
+
speakerDisabled: { type: Boolean, default: false },
|
|
9
10
|
compact: { type: Boolean, default: false },
|
|
10
|
-
|
|
11
|
+
iconOnly: { type: Boolean, default: false },
|
|
12
|
+
reviewInTranscript: { type: Boolean, default: false },
|
|
13
|
+
toolsTarget: { type: Object, default: null },
|
|
14
|
+
feedbackTarget: { type: Object, default: null }
|
|
11
15
|
});
|
|
12
16
|
const {
|
|
13
17
|
targetLabel, voice, error, microphoneMuted, live, starting, callMode,
|
|
@@ -17,13 +21,14 @@ const {
|
|
|
17
21
|
deliverTranscript, discardHeldRecording, editTranscript
|
|
18
22
|
} = props.session;
|
|
19
23
|
const controlStatus = computed(() => {
|
|
24
|
+
if (props.iconOnly && voice.listening.value && !starting.value && !error.value && !voice.error.value) return "";
|
|
20
25
|
if (!props.compact || !props.session.hasUnsentSpeech.value) return callStatus.value;
|
|
21
|
-
if (pendingTranscript.value) return "Send or discard speech to send text.";
|
|
26
|
+
if (pendingTranscript.value) return props.reviewInTranscript ? "" : "Send or discard speech to send text.";
|
|
27
|
+
if (live.value && callMode.value === "hands-free" && voice.listening.value && !microphoneMuted.value) return "Listening…";
|
|
22
28
|
return voice.listening.value ? "Recording · finish speech to send text." : "Finish speech to send text.";
|
|
23
29
|
});
|
|
24
|
-
const handsFreeListening = computed(() => live.value && callMode.value === "hands-free" && !microphoneMuted.value);
|
|
25
|
-
const canTalk = computed(() => !props.disabled && !
|
|
26
|
-
(!capturing.value || voice.listening.value || pushHolding.value || starting.value));
|
|
30
|
+
const handsFreeListening = computed(() => live.value && callMode.value === "hands-free" && voice.listening.value && !microphoneMuted.value);
|
|
31
|
+
const canTalk = computed(() => !props.disabled && (!capturing.value || voice.listening.value || pushHolding.value || starting.value));
|
|
27
32
|
const talkLabel = computed(() => starting.value ? "Connecting…" : pushHolding.value ? "Release to send" : handsFreeListening.value ? "Pause" : "Talk");
|
|
28
33
|
const gestureHint = useId();
|
|
29
34
|
const gesture = useVoiceLauncher({
|
|
@@ -36,42 +41,64 @@ const gesture = useVoiceLauncher({
|
|
|
36
41
|
|
|
37
42
|
<template>
|
|
38
43
|
<fieldset class="assistant-voice-controls" :class="{ 'assistant-voice-controls--compact': compact }" :disabled="disabled">
|
|
39
|
-
<
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
<v-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
44
|
+
<Teleport :to="feedbackTarget" :disabled="!feedbackTarget">
|
|
45
|
+
<fieldset class="assistant-voice-controls__feedback" :class="{ 'assistant-voice-controls__feedback--compact': compact }" :disabled="disabled">
|
|
46
|
+
<p
|
|
47
|
+
v-if="error || voice.error.value" class="assistant-voice__error"
|
|
48
|
+
:class="{ 'text-body-small': compact }" role="alert"
|
|
49
|
+
>
|
|
50
|
+
{{ error || voice.error.value }}
|
|
51
|
+
</p>
|
|
52
|
+
<p v-if="controlStatus" class="assistant-voice-controls__status text-body-small" role="status" :title="callStatus">{{ controlStatus }}</p>
|
|
53
|
+
<p v-if="compact && !reviewInTranscript && heldTranscript && !pendingTranscript" class="assistant-voice-controls__provisional" role="status" aria-label="Recognized words">Not sent · {{ heldTranscript }}</p>
|
|
54
|
+
<p v-if="readAloud && voice.playbackBlocked?.value" class="assistant-voice-controls__status" role="status">Sound is blocked. Enable sound to hear future replies.</p>
|
|
55
|
+
<v-btn v-if="readAloud && voice.playbackBlocked?.value" min-height="48" @click="enableSound">Enable sound</v-btn>
|
|
56
|
+
<section v-if="!reviewInTranscript && pendingTranscript && (pendingTranscript.reviewBeforeSend || !sending)" class="assistant-voice__review" aria-label="Review voice message">
|
|
57
|
+
<v-textarea
|
|
58
|
+
class="assistant-voice__review-input" :model-value="pendingTranscript.text" label="Review your message" :disabled="sending"
|
|
59
|
+
:density="compact ? 'compact' : 'default'" :rows="compact ? 1 : 2" :max-rows="compact ? 2 : 3"
|
|
60
|
+
auto-grow hide-details @update:model-value="editTranscript"
|
|
61
|
+
/>
|
|
62
|
+
<div class="assistant-voice__review-actions">
|
|
63
|
+
<v-btn :disabled="sending" min-height="48" @click="discardHeldRecording">Discard</v-btn>
|
|
64
|
+
<v-btn :disabled="sending || !pendingTranscript.text.trim()" min-height="48" color="primary" @click="deliverTranscript">{{ sending ? 'Sending…' : 'Send' }}</v-btn>
|
|
65
|
+
</div>
|
|
66
|
+
</section>
|
|
67
|
+
</fieldset>
|
|
68
|
+
</Teleport>
|
|
55
69
|
<Teleport :to="toolsTarget" :disabled="!toolsTarget">
|
|
56
|
-
<fieldset class="assistant-voice-controls__tools" :class="{ 'assistant-voice-controls__tools--compact': compact }" :disabled="disabled" :aria-label="`${targetLabel} voice chat controls`">
|
|
57
|
-
<
|
|
70
|
+
<fieldset class="assistant-voice-controls__tools" :class="{ 'assistant-voice-controls__tools--compact': compact, 'assistant-voice-controls__tools--icons': iconOnly }" :disabled="disabled" :aria-label="`${targetLabel} voice chat controls`">
|
|
71
|
+
<div v-if="compact || iconOnly || speechActive" class="assistant-voice-controls__stop">
|
|
72
|
+
<v-btn v-if="speechActive" :icon="mdiStop" variant="text" min-width="48" min-height="48" :aria-label="`Stop ${targetLabel} speaking`" title="Stop speaking" @click="stopSpeech" />
|
|
73
|
+
</div>
|
|
58
74
|
<slot name="work-control" />
|
|
59
75
|
<div class="assistant-voice__talk-action">
|
|
60
76
|
<v-btn
|
|
61
|
-
class="assistant-voice__talk" color="primary" :variant="handsFreeListening ? 'tonal' : 'flat'" :min-height="compact ? 48 : 64" rounded="xl"
|
|
62
|
-
:
|
|
63
|
-
:
|
|
77
|
+
class="assistant-voice__talk" color="primary" :variant="iconOnly ? 'text' : handsFreeListening ? 'tonal' : 'flat'" :min-height="iconOnly ? 44 : compact ? 48 : 64" rounded="xl"
|
|
78
|
+
:icon="iconOnly ? (voice.listening.value && !microphoneMuted ? mdiMicrophone : mdiMicrophoneOff) : undefined"
|
|
79
|
+
:prepend-icon="iconOnly ? undefined : (voice.listening.value && !microphoneMuted ? mdiMicrophone : mdiMicrophoneOff)"
|
|
80
|
+
:aria-label="iconOnly ? talkLabel : undefined" title="click or long press to talk"
|
|
81
|
+
:aria-pressed="handsFreeListening || pushHolding" :aria-describedby="gestureHint"
|
|
82
|
+
:disabled="!canTalk" :slim="compact || iconOnly"
|
|
83
|
+
:width="iconOnly ? 40 : compact ? 148 : undefined" :min-width="iconOnly ? 40 : compact ? 148 : undefined"
|
|
64
84
|
@pointerdown="gesture.pointerDown" @pointerup="gesture.pointerUp" @pointercancel="gesture.cancel" @lostpointercapture="gesture.cancel"
|
|
65
85
|
@keydown.space.prevent="gesture.keyDown" @keyup.space.prevent="gesture.keyUp"
|
|
66
86
|
@keydown.enter.prevent="gesture.keyDown" @keyup.enter.prevent="gesture.keyUp" @blur="gesture.cancel" @contextmenu.prevent @click="gesture.click"
|
|
67
87
|
>
|
|
68
|
-
|
|
88
|
+
<span v-if="iconOnly" class="assistant-voice-controls__icon-disc"><v-icon size="20" :icon="voice.listening.value && !microphoneMuted ? mdiMicrophone : mdiMicrophoneOff" /></span>
|
|
89
|
+
<template v-else>{{ talkLabel }}</template>
|
|
69
90
|
</v-btn>
|
|
70
91
|
<v-btn
|
|
71
|
-
:icon="readAloud ? mdiVolumeHigh : mdiVolumeOff" :aria-pressed="readAloud" variant="tonal" :min-height="compact ? 48 : 64" :min-width="compact ? 48 : 64" rounded="xl"
|
|
92
|
+
:icon="readAloud ? mdiVolumeHigh : mdiVolumeOff" :aria-pressed="readAloud" :variant="iconOnly ? 'text' : 'tonal'" :color="iconOnly ? 'primary' : undefined" :min-height="iconOnly ? 44 : compact ? 48 : 64" :width="iconOnly ? 40 : undefined" :min-width="iconOnly ? 40 : compact ? 48 : 64" rounded="xl"
|
|
72
93
|
:aria-label="readAloud ? `Turn ${targetLabel} read-aloud off` : `Read ${targetLabel} answers aloud`"
|
|
73
|
-
:title="readAloud ? 'Spoken replies on' : 'Spoken replies off'"
|
|
74
|
-
|
|
94
|
+
:title="readAloud ? 'Spoken replies on' : 'Spoken replies off'"
|
|
95
|
+
:disabled="speakerDisabled || session.readAloudChangePending?.value === true" @click="toggleReadAloud"
|
|
96
|
+
>
|
|
97
|
+
<template v-if="iconOnly" #default>
|
|
98
|
+
<span class="assistant-voice-controls__icon-disc"><v-icon size="20" :icon="readAloud ? mdiVolumeHigh : mdiVolumeOff" /></span>
|
|
99
|
+
</template>
|
|
100
|
+
</v-btn>
|
|
101
|
+
<slot name="settings-control" />
|
|
75
102
|
<p :id="gestureHint" class="text-body-small text-medium-emphasis">{{ handsFreeListening ? 'Tap to pause · hold to speak' : 'Tap for hands-free · hold to speak' }}</p>
|
|
76
103
|
</div>
|
|
77
104
|
</fieldset>
|
|
@@ -80,11 +107,18 @@ const gesture = useVoiceLauncher({
|
|
|
80
107
|
</template>
|
|
81
108
|
|
|
82
109
|
<style scoped>
|
|
83
|
-
.assistant-voice-controls, .assistant-voice-controls__tools { border: 0; padding: 0; margin: 0; min-width: 0; }
|
|
110
|
+
.assistant-voice-controls, .assistant-voice-controls__feedback, .assistant-voice-controls__tools { border: 0; padding: 0; margin: 0; min-width: 0; }
|
|
84
111
|
.assistant-voice-controls { flex: 0 0 auto; }
|
|
85
112
|
.assistant-voice-controls--compact { flex: 1 1 100%; width: 100%; }
|
|
86
113
|
.assistant-voice-controls__tools { display: flex; align-items: center; justify-content: center; flex-wrap: wrap; }
|
|
87
114
|
.assistant-voice-controls__status { text-align: center; margin: 0; }
|
|
115
|
+
.assistant-voice-controls__feedback--compact > .assistant-voice-controls__status { min-height: 1lh; }
|
|
116
|
+
.assistant-voice-controls__stop { display: contents; }
|
|
117
|
+
.assistant-voice-controls__tools--compact .assistant-voice-controls__stop {
|
|
118
|
+
display: block;
|
|
119
|
+
flex: 0 0 48px;
|
|
120
|
+
height: 48px;
|
|
121
|
+
}
|
|
88
122
|
.assistant-voice-controls__provisional { margin: 0; max-height: 3rem; overflow-y: auto; white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
89
123
|
.assistant-voice__talk-action { width: min(288px, 100%); display: grid; grid-template-columns: minmax(0, 1fr) 64px; gap: 6px 8px; padding: 16px; }
|
|
90
124
|
.assistant-voice__talk { touch-action: none; user-select: none; -webkit-touch-callout: none; }
|
|
@@ -92,10 +126,24 @@ const gesture = useVoiceLauncher({
|
|
|
92
126
|
.assistant-voice__review { flex: 0 0 auto; max-height: 160px; overflow-y: auto; padding: 8px 16px; }
|
|
93
127
|
.assistant-voice__review-actions { display: flex; justify-content: flex-end; gap: 8px; }
|
|
94
128
|
.assistant-voice__error { padding: 8px 16px; color: rgb(var(--v-theme-error)); }
|
|
95
|
-
.assistant-voice-controls__tools--compact { justify-content: flex-start; gap:
|
|
96
|
-
.assistant-voice-controls__tools--compact .assistant-voice__talk-action { display: flex; padding: 0; width: auto; gap:
|
|
129
|
+
.assistant-voice-controls__tools--compact { justify-content: flex-start; gap: 2px; }
|
|
130
|
+
.assistant-voice-controls__tools--compact .assistant-voice__talk-action { display: flex; padding: 0; width: auto; gap: 0; }
|
|
97
131
|
.assistant-voice-controls__tools--compact .assistant-voice__talk-action p { position: absolute; width: 1px; height: 1px; padding: 0; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
|
|
98
|
-
.assistant-voice-
|
|
99
|
-
.assistant-voice-
|
|
132
|
+
.assistant-voice-controls__tools--icons { position: relative; width: max-content; flex-wrap: nowrap; justify-content: flex-end; gap: 0; }
|
|
133
|
+
.assistant-voice-controls__tools--icons .assistant-voice-controls__stop { position: absolute; right: calc(100% + 2px); top: 0; width: 48px; height: 48px; pointer-events: none; }
|
|
134
|
+
.assistant-voice-controls__tools--icons .assistant-voice-controls__stop > .v-btn { pointer-events: auto; }
|
|
135
|
+
.assistant-voice-controls__icon-disc { display: flex; align-items: center; justify-content: center; width: 32px; height: 32px; border-radius: 50%; background: rgba(var(--v-theme-primary), .12); }
|
|
136
|
+
.assistant-voice-controls__tools--icons .assistant-voice__talk-action { display: flex; padding: 0; width: auto; gap: 0; }
|
|
137
|
+
.assistant-voice-controls__tools--icons .assistant-voice__talk-action p { position: absolute; width: 1px; height: 1px; padding: 0; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
|
|
138
|
+
.assistant-voice-controls__feedback--compact .assistant-voice__review {
|
|
139
|
+
display: flex;
|
|
140
|
+
align-items: flex-end;
|
|
141
|
+
gap: 4px;
|
|
142
|
+
max-height: 128px;
|
|
143
|
+
padding: 0;
|
|
144
|
+
}
|
|
145
|
+
.assistant-voice-controls__feedback--compact .assistant-voice__review-input { flex: 1 1 auto; min-width: 0; }
|
|
146
|
+
.assistant-voice-controls__feedback--compact .assistant-voice__review-actions { flex: 0 0 auto; }
|
|
147
|
+
.assistant-voice-controls__feedback--compact .assistant-voice__error { margin: 0; padding: 0; }
|
|
100
148
|
@media (max-height: 650px) { .assistant-voice__talk-action { padding-top: 8px; } }
|
|
101
149
|
</style>
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
<script setup>
|
|
2
|
-
import {
|
|
2
|
+
import { mdiHeadset, mdiStop } from "@mdi/js";
|
|
3
3
|
import ConversationDialog from "./ConversationDialog.vue";
|
|
4
4
|
import VoiceConversation from "./VoiceConversation.vue";
|
|
5
|
+
import VoiceConversationSettings from "./VoiceConversationSettings.vue";
|
|
5
6
|
|
|
6
|
-
const props = defineProps({
|
|
7
|
+
const props = defineProps({
|
|
8
|
+
controller: { type: Object, required: true }, activator: { type: Object, default: null },
|
|
9
|
+
avatarSize: { type: String, default: "hidden" },
|
|
10
|
+
presentation: { type: String, default: "dialog", validator: value => ["dialog", "inline"].includes(value) }
|
|
11
|
+
});
|
|
7
12
|
defineEmits(["update:avatarSize"]);
|
|
8
13
|
const state = props.controller.state;
|
|
9
14
|
async function invoke(operation) {
|
|
@@ -20,31 +25,16 @@ async function openText() {
|
|
|
20
25
|
|
|
21
26
|
<template>
|
|
22
27
|
<ConversationDialog
|
|
23
|
-
v-if="state.session" :model-value="state.visible" :activator="activator"
|
|
28
|
+
v-if="state.session" :model-value="state.visible" :activator="activator" :presentation="presentation"
|
|
24
29
|
:title="state.binding.label || 'Assistant'" mode="talk"
|
|
25
30
|
:show-modes="Boolean(state.binding.openText && !state.binding.adapter)"
|
|
26
|
-
minimizable close-label="Close voice chat" @minimize="controller.minimize()" @update:model-value="invoke(() => controller.end({ discard: true }))"
|
|
31
|
+
:minimizable="presentation === 'dialog'" close-label="Close voice chat" @minimize="controller.minimize()" @update:model-value="invoke(() => controller.end({ discard: true }))"
|
|
27
32
|
@update:mode="value => value === 'text' && invoke(openText)"
|
|
28
33
|
>
|
|
29
|
-
<template #header-actions>
|
|
30
|
-
<
|
|
31
|
-
<template #
|
|
32
|
-
|
|
33
|
-
</template>
|
|
34
|
-
<v-card class="voice-host__settings" role="region" aria-label="Voice settings">
|
|
35
|
-
<v-card-text>
|
|
36
|
-
<slot name="settings" :voice="state.session.voice">
|
|
37
|
-
<v-select
|
|
38
|
-
v-model="state.session.voice.selectedVoice.value"
|
|
39
|
-
:items="state.session.voice.availableVoices.value" item-title="label" item-value="id"
|
|
40
|
-
:disabled="state.session.voice.availableVoices.value.length < 2"
|
|
41
|
-
label="Speaking voice" density="comfortable"
|
|
42
|
-
hint="Applies to the next spoken reply" persistent-hint
|
|
43
|
-
/>
|
|
44
|
-
</slot>
|
|
45
|
-
</v-card-text>
|
|
46
|
-
</v-card>
|
|
47
|
-
</v-menu>
|
|
34
|
+
<template v-if="!state.binding.adapter" #header-actions>
|
|
35
|
+
<VoiceConversationSettings :voice="state.session.voice" @error="state.error = $event.message">
|
|
36
|
+
<template v-if="$slots.settings" #default="settings"><slot name="settings" v-bind="settings" /></template>
|
|
37
|
+
</VoiceConversationSettings>
|
|
48
38
|
</template>
|
|
49
39
|
<v-alert v-if="state.error" type="error" density="compact" role="alert">{{ state.error }}</v-alert>
|
|
50
40
|
<v-alert v-if="state.nextTarget" type="info" class="voice-host__switch">
|
|
@@ -63,6 +53,12 @@ async function openText() {
|
|
|
63
53
|
:avatar-size="avatarSize" @update:avatar-size="$emit('update:avatarSize', $event)"
|
|
64
54
|
>
|
|
65
55
|
<template v-if="$slots.avatar" #avatar="visual"><slot name="avatar" v-bind="visual" :binding="state.binding" /></template>
|
|
56
|
+
<template v-if="$slots['avatar-control']" #avatar-control="scope"><slot name="avatar-control" v-bind="scope" :binding="state.binding" :session="state.session" /></template>
|
|
57
|
+
<template v-if="presentation === 'inline' || state.binding.adapter" #settings-control>
|
|
58
|
+
<VoiceConversationSettings :voice="state.session.voice" compact @error="state.error = $event.message">
|
|
59
|
+
<template v-if="$slots.settings" #default="settings"><slot name="settings" v-bind="settings" /></template>
|
|
60
|
+
</VoiceConversationSettings>
|
|
61
|
+
</template>
|
|
66
62
|
<template #work-control>
|
|
67
63
|
<v-btn v-if="state.binding.cancelWork && state.binding.state.status === 'working'" :disabled="state.busy" :icon="mdiStop" variant="text" aria-label="Stop agent work" title="Stop agent work" @click="invoke(state.binding.cancelWork)" />
|
|
68
64
|
</template>
|
|
@@ -70,7 +66,7 @@ async function openText() {
|
|
|
70
66
|
</slot>
|
|
71
67
|
</div>
|
|
72
68
|
</ConversationDialog>
|
|
73
|
-
<template v-if="state.session && !state.visible">
|
|
69
|
+
<template v-if="presentation === 'dialog' && state.session && !state.visible">
|
|
74
70
|
<slot v-if="$slots.reopen" name="reopen" :controller="controller" />
|
|
75
71
|
<v-btn v-else class="voice-host__reopen" color="primary" :prepend-icon="mdiHeadset" min-height="56" @click="controller.reveal()">
|
|
76
72
|
{{ state.binding.label || 'Assistant' }} · Voice chat
|
|
@@ -82,6 +78,5 @@ async function openText() {
|
|
|
82
78
|
.voice-host__body { display: flex; flex: 1; min-height: 0; overflow: hidden; }
|
|
83
79
|
.voice-host__switch-actions { display: flex; flex-wrap: wrap; gap: 4px; }
|
|
84
80
|
.voice-host__switch { flex: 0 0 auto; }
|
|
85
|
-
.voice-host__settings { width: 320px; max-width: calc(100vw - 32px); }
|
|
86
81
|
.voice-host__reopen { position: fixed; right: 12px; bottom: max(12px, env(safe-area-inset-bottom)); max-width: calc(100vw - 24px); z-index: 1900; }
|
|
87
82
|
</style>
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
<script setup>
|
|
2
|
+
import { mdiCog } from "@mdi/js";
|
|
3
|
+
import { useSlots } from "vue";
|
|
4
|
+
|
|
5
|
+
const props = defineProps({
|
|
6
|
+
voice: { type: Object, default: null },
|
|
7
|
+
compact: Boolean
|
|
8
|
+
});
|
|
9
|
+
const emit = defineEmits(["error"]);
|
|
10
|
+
const slots = useSlots();
|
|
11
|
+
function selectVoice(value) {
|
|
12
|
+
const selectedVoice = props.voice.selectedVoice;
|
|
13
|
+
selectedVoice.value = value;
|
|
14
|
+
}
|
|
15
|
+
async function openSettings(open) {
|
|
16
|
+
if (!open || slots.default || !props.voice) return;
|
|
17
|
+
try { await props.voice.connect(); }
|
|
18
|
+
catch (error) { emit("error", error); }
|
|
19
|
+
}
|
|
20
|
+
</script>
|
|
21
|
+
|
|
22
|
+
<template>
|
|
23
|
+
<v-menu :close-on-content-click="false" location="bottom end" @update:model-value="openSettings">
|
|
24
|
+
<template #activator="{ props: settingsButton }">
|
|
25
|
+
<v-btn
|
|
26
|
+
v-bind="settingsButton" :icon="mdiCog" variant="text" :color="compact ? 'primary' : undefined"
|
|
27
|
+
:width="compact ? 40 : undefined" :min-width="compact ? 40 : 48" :min-height="compact ? 44 : 48" :disabled="!voice && !$slots.default"
|
|
28
|
+
aria-label="Voice settings" title="Voice settings"
|
|
29
|
+
>
|
|
30
|
+
<template v-if="compact" #default><span class="voice-settings__disc"><v-icon size="20" :icon="mdiCog" /></span></template>
|
|
31
|
+
</v-btn>
|
|
32
|
+
</template>
|
|
33
|
+
<v-card class="voice-settings__menu" role="region" aria-label="Voice settings">
|
|
34
|
+
<v-card-text>
|
|
35
|
+
<slot :voice="voice">
|
|
36
|
+
<v-select
|
|
37
|
+
v-if="voice" :model-value="voice.selectedVoice.value" @update:model-value="selectVoice"
|
|
38
|
+
:items="voice.availableVoices.value" item-title="label" item-value="id"
|
|
39
|
+
:disabled="voice.availableVoices.value.length < 2"
|
|
40
|
+
label="Speaking voice" density="comfortable"
|
|
41
|
+
hint="Applies to the next spoken reply" persistent-hint
|
|
42
|
+
/>
|
|
43
|
+
</slot>
|
|
44
|
+
</v-card-text>
|
|
45
|
+
</v-card>
|
|
46
|
+
</v-menu>
|
|
47
|
+
</template>
|
|
48
|
+
|
|
49
|
+
<style scoped>
|
|
50
|
+
.voice-settings__menu { width: 320px; max-width: calc(100vw - 32px); }
|
|
51
|
+
.voice-settings__disc { display: inline-flex; align-items: center; justify-content: center; width: 32px; height: 32px; border-radius: 50%; background: rgba(var(--v-theme-primary), .12); }
|
|
52
|
+
</style>
|
package/src/client/index.js
CHANGED
|
@@ -4,6 +4,7 @@ export { default as VoiceConversationHost } from "./VoiceConversationHost.vue";
|
|
|
4
4
|
export { default as ConversationDialog } from "./ConversationDialog.vue";
|
|
5
5
|
export { default as VoiceConversation } from "./VoiceConversation.vue";
|
|
6
6
|
export { default as VoiceConversationControls } from "./VoiceConversationControls.vue";
|
|
7
|
+
export { default as VoiceConversationSettings } from "./VoiceConversationSettings.vue";
|
|
7
8
|
|
|
8
9
|
export { useVoiceLauncher } from "./voiceLauncher.js";
|
|
9
10
|
export { useVoiceAvatar } from "./voiceAvatar.js";
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { computed, onScopeDispose, ref, watch } from "vue";
|
|
2
2
|
import { useVoiceTransport } from "./voiceTransport.js";
|
|
3
3
|
import { takeStreamingSpeech } from "../shared/protocol.js";
|
|
4
|
+
import { createConversationNarrationTracker } from "./conversationNarration.js";
|
|
4
5
|
|
|
5
6
|
/** Created inside the controller's effect scope, independent of a mounted screen. */
|
|
6
7
|
export function useVoiceConversation(binding, { socketUrl, createTransport = useVoiceTransport } = {}) {
|
|
7
8
|
const oneOffTalkMode = computed(() => binding.defaults?.talkMode === "hold" ? "hold" : "tap");
|
|
8
9
|
const targetLabel = computed(() => binding.label || "Assistant");
|
|
9
10
|
const readAloud = ref(binding.defaults?.readAloud === true);
|
|
11
|
+
const readAloudChangePending = computed(() => binding.readAloudChangePending === true);
|
|
10
12
|
const soundPreparing = ref(false);
|
|
11
13
|
let soundRevision = 0;
|
|
12
14
|
const conversationId = binding.conversationId || binding.id;
|
|
@@ -15,6 +17,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
15
17
|
const heldReview = ref(false);
|
|
16
18
|
const holding = ref(false);
|
|
17
19
|
const sending = ref(false);
|
|
20
|
+
const takingTranscript = ref(false);
|
|
18
21
|
const live = ref(false);
|
|
19
22
|
const starting = ref(false);
|
|
20
23
|
const callMode = ref(binding.defaults?.mode === "hands-free" ? "hands-free" : "push-to-talk");
|
|
@@ -109,6 +112,72 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
109
112
|
let requestedMessageId = "";
|
|
110
113
|
let speechInvitation = "";
|
|
111
114
|
let lastStreamingId = "";
|
|
115
|
+
const narrationTracker = createConversationNarrationTracker();
|
|
116
|
+
const narrationPageVisible = ref(globalThis.document?.visibilityState !== "hidden");
|
|
117
|
+
const hiddenNarrationPage = computed(() => Boolean(binding.narration) && !narrationPageVisible.value);
|
|
118
|
+
let narrationSettleTimer;
|
|
119
|
+
let thinkingMurmurTimer;
|
|
120
|
+
let narrationEpoch = 0;
|
|
121
|
+
|
|
122
|
+
function clearNarrationTimers() {
|
|
123
|
+
++narrationEpoch;
|
|
124
|
+
clearTimeout(narrationSettleTimer);
|
|
125
|
+
clearTimeout(thinkingMurmurTimer);
|
|
126
|
+
}
|
|
127
|
+
function retireNarration(selected = () => true) {
|
|
128
|
+
for (let index = speechQueue.length - 1; index >= 0; --index) {
|
|
129
|
+
const reply = speechQueue[index];
|
|
130
|
+
if (!reply.narrationKind || !selected(reply.narrationKind)) continue;
|
|
131
|
+
cancelReply(reply, "narration-retired");
|
|
132
|
+
speechQueue.splice(index, 1);
|
|
133
|
+
if (reply.turnId && voice.activeSpeechTurnId.value === reply.turnId) voice.stopSpeaking("narration-retired");
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function narrationEnabled() {
|
|
137
|
+
const narration = binding.narration;
|
|
138
|
+
return Boolean(narration && narration.eligible && !narration.loading && readAloud.value
|
|
139
|
+
&& binding.state.available !== false && narrationPageVisible.value && !capturing.value
|
|
140
|
+
&& !starting.value && !speechSuppressed && !disposed);
|
|
141
|
+
}
|
|
142
|
+
function queueNarration(kind, text) {
|
|
143
|
+
// Optional activity uses the original queue but never impersonates a canonical output.
|
|
144
|
+
if (!narrationEnabled() || speechQueue.some(reply => !reply.narrationKind)
|
|
145
|
+
|| speechActive.value && !speechQueue.some(reply => reply.narrationKind && reply.turnId === voice.activeSpeechTurnId.value)) return;
|
|
146
|
+
speechQueue.push({ narrationKind: kind, raw: text, consumed: 0, final: true, started: false });
|
|
147
|
+
void drainSpeech();
|
|
148
|
+
}
|
|
149
|
+
function observeNarration(settled = false, enabled = narrationEnabled()) {
|
|
150
|
+
const narration = binding.narration;
|
|
151
|
+
if (!narration) return;
|
|
152
|
+
const entries = narrationTracker.observe(narration.turns, {
|
|
153
|
+
includeFinals: false, settled, enabled,
|
|
154
|
+
vocalizeThinking: narration.vocalizeThinking,
|
|
155
|
+
vocalizeInterimTurns: narration.vocalizeInterimTurns
|
|
156
|
+
});
|
|
157
|
+
for (const entry of entries) queueNarration(entry.kind, entry.text);
|
|
158
|
+
}
|
|
159
|
+
function scheduleThinkingMurmur(epoch) {
|
|
160
|
+
const narration = binding.narration;
|
|
161
|
+
if (!narrationEnabled() || !narration.working || !narration.thinkingSounds) return;
|
|
162
|
+
thinkingMurmurTimer = setTimeout(() => {
|
|
163
|
+
if (epoch !== narrationEpoch || !narrationEnabled() || !binding.narration?.working || !binding.narration.thinkingSounds) return;
|
|
164
|
+
if (!speechQueue.length && !draining && !speechActive.value && voice.state.value === "idle") queueNarration("murmur", "Hmm...");
|
|
165
|
+
scheduleThinkingMurmur(epoch);
|
|
166
|
+
}, 20_000);
|
|
167
|
+
}
|
|
168
|
+
function refreshNarration(settled = false) {
|
|
169
|
+
clearNarrationTimers();
|
|
170
|
+
observeNarration(settled);
|
|
171
|
+
const narration = binding.narration;
|
|
172
|
+
if (!narrationEnabled() || !narration.working) return;
|
|
173
|
+
const epoch = narrationEpoch;
|
|
174
|
+
if (narration.vocalizeThinking || narration.vocalizeInterimTurns) {
|
|
175
|
+
narrationSettleTimer = setTimeout(() => {
|
|
176
|
+
if (epoch === narrationEpoch && narrationEnabled()) observeNarration(true);
|
|
177
|
+
}, 700);
|
|
178
|
+
}
|
|
179
|
+
scheduleThinkingMurmur(epoch);
|
|
180
|
+
}
|
|
112
181
|
|
|
113
182
|
function observe(callback, value) {
|
|
114
183
|
try {
|
|
@@ -121,16 +190,18 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
121
190
|
if (reply.terminal || (phase === "started" && reply.audible)) return;
|
|
122
191
|
if (phase === "started") reply.audible = true;
|
|
123
192
|
else reply.terminal = true;
|
|
193
|
+
if (reply.narrationKind) return;
|
|
124
194
|
observe(binding.onPlayback, {
|
|
125
195
|
conversationId, outputId: reply.outputId, phase, ...(reason ? { reason } : {})
|
|
126
196
|
});
|
|
127
197
|
}
|
|
128
198
|
function receivePlayback(event) {
|
|
129
|
-
const reply = [...replies.values()].find(reply => reply.turnId === event.turnId)
|
|
199
|
+
const reply = [...replies.values()].find(reply => reply.turnId === event.turnId)
|
|
200
|
+
|| speechQueue.find(reply => reply.narrationKind && reply.turnId === event.turnId);
|
|
130
201
|
if (!reply) return;
|
|
131
202
|
if (event.phase === "completed") {
|
|
132
203
|
reply.drained = true;
|
|
133
|
-
if (reply.canonicalFinal) emitReply(reply, "completed");
|
|
204
|
+
if (reply.canonicalFinal || reply.narrationKind) emitReply(reply, "completed");
|
|
134
205
|
} else emitReply(reply, event.phase, event.reason);
|
|
135
206
|
}
|
|
136
207
|
function cancelReply(reply, reason = "cancelled", phase = "interrupted") {
|
|
@@ -139,6 +210,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
139
210
|
}
|
|
140
211
|
function stopSpeech() { cancelSpeech(); }
|
|
141
212
|
function cancelSpeech(reason = "cancelled", phase = "interrupted") {
|
|
213
|
+
clearNarrationTimers();
|
|
214
|
+
retireNarration();
|
|
142
215
|
for (const reply of replies.values()) {
|
|
143
216
|
if (!reply.terminal) cancelReply(reply, reason, phase);
|
|
144
217
|
}
|
|
@@ -180,7 +253,11 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
180
253
|
reply.started = true;
|
|
181
254
|
reply.turnId = crypto.randomUUID();
|
|
182
255
|
const accepted = await voice.speak(chunk.text, reply.turnId, { stream: true });
|
|
183
|
-
if (
|
|
256
|
+
if (disposed) return;
|
|
257
|
+
if (reply.cancelled) {
|
|
258
|
+
if (reply.narrationKind) continue;
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
184
261
|
if (accepted === false) {
|
|
185
262
|
cancelReply(reply, "no-audio");
|
|
186
263
|
continue;
|
|
@@ -195,6 +272,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
195
272
|
let reply = replies.get(message.id);
|
|
196
273
|
if (!reply) {
|
|
197
274
|
if (!primed || !readAloud.value || speechSuppressed || seen.has(message.id) || (!final && message.autonomous)) return;
|
|
275
|
+
retireNarration();
|
|
198
276
|
reply = {
|
|
199
277
|
id: message.id, outputId: message.outputId || message.id, streamId: message.streamId,
|
|
200
278
|
raw: "", consumed: 0, final: false, canonicalFinal: false, started: false
|
|
@@ -203,6 +281,14 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
203
281
|
speechQueue.push(reply);
|
|
204
282
|
}
|
|
205
283
|
if (reply.cancelled) return;
|
|
284
|
+
if (hiddenNarrationPage.value && !reply.canonicalFinal && (final || !reply.raw)) {
|
|
285
|
+
// Consume hidden outputs without replay; a streaming answer cannot claim
|
|
286
|
+
// completion when its first canonical final was not admitted for speech.
|
|
287
|
+
cancelReply(reply, "page-hidden");
|
|
288
|
+
if (reply.turnId && voice.activeSpeechTurnId.value === reply.turnId) voice.stopSpeaking("page-hidden");
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
if (final) retireNarration();
|
|
206
292
|
const text = String(message.text || "").slice(0, 4000);
|
|
207
293
|
const spokenPrefix = reply.raw.slice(0, reply.consumed);
|
|
208
294
|
if (!text.startsWith(spokenPrefix) && !(final && text.trimEnd() === spokenPrefix.trimEnd())) {
|
|
@@ -233,9 +319,13 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
233
319
|
}
|
|
234
320
|
}
|
|
235
321
|
async function toggleReadAloud() {
|
|
322
|
+
if (readAloudChangePending.value) return;
|
|
236
323
|
error.value = "";
|
|
237
324
|
readAloud.value = !readAloud.value;
|
|
238
325
|
observe(binding.onReadAloudChange, readAloud.value);
|
|
326
|
+
await applyReadAloud();
|
|
327
|
+
}
|
|
328
|
+
async function applyReadAloud() {
|
|
239
329
|
if (!readAloud.value) {
|
|
240
330
|
++soundRevision;
|
|
241
331
|
soundPreparing.value = false;
|
|
@@ -245,7 +335,13 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
245
335
|
speechSuppressed = false;
|
|
246
336
|
speechInvitation = "";
|
|
247
337
|
await enableSound();
|
|
338
|
+
if (!disposed) refreshNarration();
|
|
248
339
|
}
|
|
340
|
+
watch(() => binding.defaults?.readAloud, (value) => {
|
|
341
|
+
if (disposed || typeof value !== "boolean" || value === readAloud.value) return;
|
|
342
|
+
readAloud.value = value;
|
|
343
|
+
void applyReadAloud();
|
|
344
|
+
});
|
|
249
345
|
async function deliverTranscript() {
|
|
250
346
|
const pending = pendingTranscript.value;
|
|
251
347
|
if (!pending || sending.value) return;
|
|
@@ -263,6 +359,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
263
359
|
startupAbort?.abort();
|
|
264
360
|
startupAbort = null;
|
|
265
361
|
starting.value = false;
|
|
362
|
+
releaseEditCapturePause();
|
|
266
363
|
recording = null;
|
|
267
364
|
await voice.cancelListening();
|
|
268
365
|
}
|
|
@@ -273,11 +370,21 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
273
370
|
await cancelRecording();
|
|
274
371
|
}
|
|
275
372
|
function dismissTranscript() {
|
|
373
|
+
releaseEditCapturePause();
|
|
276
374
|
pendingTranscript.value = null;
|
|
277
375
|
heldReview.value = false;
|
|
278
376
|
holding.value = false;
|
|
279
377
|
error.value = "";
|
|
280
378
|
}
|
|
379
|
+
function releaseEditCapturePause() {
|
|
380
|
+
const pending = pendingTranscript.value;
|
|
381
|
+
const captureId = pending?.editPausedCaptureId;
|
|
382
|
+
if (!captureId) return;
|
|
383
|
+
delete pending.editPausedCaptureId;
|
|
384
|
+
if (recording?.messageId === captureId && !disposed && live.value && callMode.value === "hands-free") {
|
|
385
|
+
voice.setMicrophoneMuted(false);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
281
388
|
async function talk() {
|
|
282
389
|
error.value = "";
|
|
283
390
|
try {
|
|
@@ -288,7 +395,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
288
395
|
} catch (cause) { recording = null; error.value = cause.message; }
|
|
289
396
|
}
|
|
290
397
|
async function startRecording(reviewBeforeSend, continuous = false) {
|
|
291
|
-
if (disposed ||
|
|
398
|
+
if (disposed || hiddenNarrationPage.value || starting.value || microphoneMuted.value || capturing.value || committing ||
|
|
399
|
+
(pendingTranscript.value || sending.value) && !(continuous && live.value && (callMode.value === "hands-free" || pushHolding.value))) return;
|
|
292
400
|
error.value = "";
|
|
293
401
|
reviewBeforeSend ||= binding.defaults?.reviewBeforeSend === true;
|
|
294
402
|
heldReview.value = reviewBeforeSend;
|
|
@@ -302,10 +410,13 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
302
410
|
} catch (cause) { if (recording === next) { recording = null; error.value = cause.message; } }
|
|
303
411
|
}
|
|
304
412
|
async function toggleMicrophoneMuted() {
|
|
413
|
+
// A person's explicit microphone choice supersedes an Edit-owned pause.
|
|
414
|
+
if (pendingTranscript.value) delete pendingTranscript.value.editPausedCaptureId;
|
|
305
415
|
voice.setMicrophoneMuted(!microphoneMuted.value);
|
|
416
|
+
if (!microphoneMuted.value && recording) delete recording.finishAfterPending;
|
|
306
417
|
clearTimeout(stopTimer);
|
|
307
418
|
if (microphoneMuted.value && starting.value) await cancelRecording();
|
|
308
|
-
else if (!microphoneMuted.value && live.value && callMode.value === "hands-free" && !capturing.value
|
|
419
|
+
else if (!microphoneMuted.value && live.value && callMode.value === "hands-free" && !capturing.value) await startRecording(false, true);
|
|
309
420
|
}
|
|
310
421
|
async function changeCallMode(mode) {
|
|
311
422
|
if (!["push-to-talk", "hands-free"].includes(mode) || mode === callMode.value || callModeBusy.value || committing) return;
|
|
@@ -317,13 +428,15 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
317
428
|
} finally { changingCallMode.value = false; }
|
|
318
429
|
}
|
|
319
430
|
async function startPushToTalk(event) {
|
|
320
|
-
if (event?.repeat || event?.isPrimary === false || event?.button !== undefined && event.button !== 0 || pushHolding.value
|
|
321
|
-
||
|
|
431
|
+
if (event?.repeat || event?.isPrimary === false || event?.button !== undefined && event.button !== 0 || pushHolding.value
|
|
432
|
+
|| disposed || hiddenNarrationPage.value || starting.value
|
|
433
|
+
|| capturing.value && !voice.listening.value || committing) return;
|
|
322
434
|
if (event?.pointerId !== undefined) event.currentTarget.setPointerCapture(event.pointerId);
|
|
323
435
|
pushInput = event?.pointerId ?? event?.key ?? null;
|
|
324
436
|
pushHolding.value = true;
|
|
325
437
|
callMode.value = "push-to-talk";
|
|
326
438
|
voice.setMicrophoneMuted(false);
|
|
439
|
+
if (recording) delete recording.finishAfterPending;
|
|
327
440
|
if (!live.value) await toggleLive();
|
|
328
441
|
if (!pushHolding.value || !live.value || disposed) return;
|
|
329
442
|
// A hold can take over an open hands-free microphone without throwing away
|
|
@@ -332,7 +445,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
332
445
|
recording.continuous = false;
|
|
333
446
|
return;
|
|
334
447
|
}
|
|
335
|
-
await startRecording(false);
|
|
448
|
+
await startRecording(false, Boolean(pendingTranscript.value || sending.value));
|
|
336
449
|
if (!recording) pushHolding.value = false;
|
|
337
450
|
}
|
|
338
451
|
async function finishPushToTalk(event) {
|
|
@@ -340,6 +453,11 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
340
453
|
pushHolding.value = false;
|
|
341
454
|
pushInput = null;
|
|
342
455
|
try {
|
|
456
|
+
if (voice.listening.value && (pendingTranscript.value || sending.value || committing)) {
|
|
457
|
+
if (recording) recording.finishAfterPending = true;
|
|
458
|
+
voice.setMicrophoneMuted(true);
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
343
461
|
if (voice.listening.value) await voice.stopListening();
|
|
344
462
|
else await cancelRecording();
|
|
345
463
|
} catch (cause) { await cancelRecording(); error.value = cause.message; }
|
|
@@ -351,7 +469,15 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
351
469
|
await cancelRecording();
|
|
352
470
|
}
|
|
353
471
|
async function toggleHandsFree() {
|
|
354
|
-
if (disposed ||
|
|
472
|
+
if (disposed || hiddenNarrationPage.value || pushHolding.value) return;
|
|
473
|
+
if (live.value && callMode.value === "hands-free" && voice.listening.value && (pendingTranscript.value || sending.value || committing)) {
|
|
474
|
+
// Pause must finish the newer recording after the existing admission,
|
|
475
|
+
// without replacing the one pending transcript or stopping output.
|
|
476
|
+
if (!microphoneMuted.value && recording) recording.finishAfterPending = true;
|
|
477
|
+
await toggleMicrophoneMuted();
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
if (committing) return;
|
|
355
481
|
if (starting.value) { await cancelRecording(); return; }
|
|
356
482
|
if (capturing.value && !voice.listening.value) return;
|
|
357
483
|
if (live.value && callMode.value === "hands-free") {
|
|
@@ -360,10 +486,16 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
360
486
|
// final sound is included; resuming starts a fresh recording.
|
|
361
487
|
await voice.stopListening();
|
|
362
488
|
voice.setMicrophoneMuted(true);
|
|
363
|
-
} else await toggleMicrophoneMuted();
|
|
489
|
+
} else if (microphoneMuted.value) await toggleMicrophoneMuted();
|
|
490
|
+
else await startRecording(false, true);
|
|
364
491
|
return;
|
|
365
492
|
}
|
|
366
|
-
if (
|
|
493
|
+
if (callMode.value === "hands-free" && (pendingTranscript.value || sending.value)) {
|
|
494
|
+
voice.setMicrophoneMuted(false);
|
|
495
|
+
await toggleLive();
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
498
|
+
if (pendingTranscript.value || sending.value || callModeBusy.value) return;
|
|
367
499
|
voice.setMicrophoneMuted(false);
|
|
368
500
|
await changeCallMode("hands-free");
|
|
369
501
|
if (!live.value) await toggleLive();
|
|
@@ -379,7 +511,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
379
511
|
await cancelRecording();
|
|
380
512
|
return;
|
|
381
513
|
}
|
|
382
|
-
if (disposed ||
|
|
514
|
+
if (disposed || hiddenNarrationPage.value || microphoneMuted.value || capturing.value || committing ||
|
|
515
|
+
(pendingTranscript.value || sending.value) && callMode.value !== "hands-free" && !pushHolding.value) return;
|
|
383
516
|
const controller = new AbortController();
|
|
384
517
|
startupAbort = controller;
|
|
385
518
|
starting.value = true;
|
|
@@ -415,7 +548,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
415
548
|
}
|
|
416
549
|
function decideTurn() {
|
|
417
550
|
const candidate = voice.endpoint.value;
|
|
418
|
-
if (!live.value || callMode.value !== "hands-free" || !voice.listening.value || microphoneMuted.value || pendingTranscript.value || sending.value || !candidate) return;
|
|
551
|
+
if (takingTranscript.value || !live.value || callMode.value !== "hands-free" || !voice.listening.value || microphoneMuted.value || pendingTranscript.value || sending.value || !candidate) return;
|
|
419
552
|
const key = `${candidate.turnId}:${candidate.revision}`;
|
|
420
553
|
if (decidedCandidate === key) return;
|
|
421
554
|
decidedCandidate = key;
|
|
@@ -441,6 +574,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
441
574
|
}
|
|
442
575
|
committing = { ...captured, turnId: candidate.turnId, revision: candidate.revision };
|
|
443
576
|
if (!voice.finishUtterance(candidate)) { committing = null; return; }
|
|
577
|
+
publishTranscript();
|
|
444
578
|
// The daemon rechecks the endpoint revision before returning final text.
|
|
445
579
|
// Ordinary steering leaves the current answer and queued audio intact.
|
|
446
580
|
} catch (cause) {
|
|
@@ -451,7 +585,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
451
585
|
}
|
|
452
586
|
}
|
|
453
587
|
}
|
|
454
|
-
watch([voice.endpoint, pendingTranscript, sending], decideTurn);
|
|
588
|
+
watch([voice.endpoint, pendingTranscript, sending, takingTranscript], decideTurn);
|
|
455
589
|
watch(voice.completedUtterance, (utterance) => {
|
|
456
590
|
if (!utterance || !live.value || !committing || committing.turnId !== utterance.turnId || committing.revision !== utterance.revision) return;
|
|
457
591
|
pendingTranscript.value = { ...committing, text: utterance.text };
|
|
@@ -462,7 +596,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
462
596
|
// A discard can lose a race with resumed speech. Keep this utterance's
|
|
463
597
|
// identity and destination until the daemon acknowledges the boundary.
|
|
464
598
|
if (reset?.turnId !== voice.activeListenTurnId.value || !live.value || !recording) return;
|
|
465
|
-
recording = { focus: { ...(binding.captureContext?.() || {}) }, messageId: crypto.randomUUID(), started: true, continuous: true
|
|
599
|
+
recording = { focus: { ...(binding.captureContext?.() || {}) }, messageId: crypto.randomUUID(), started: true, continuous: true,
|
|
600
|
+
...(recording.finishAfterPending ? { finishAfterPending: true } : {}) };
|
|
466
601
|
});
|
|
467
602
|
watch(voice.partialTranscript, (text, previous) => {
|
|
468
603
|
clearTimeout(stopTimer);
|
|
@@ -473,7 +608,16 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
473
608
|
if (live.value && !microphoneMuted.value && voice.partialTranscript.value === text) stopSpeech();
|
|
474
609
|
}, 120);
|
|
475
610
|
});
|
|
476
|
-
watch([pendingTranscript, sending], () => {
|
|
611
|
+
watch([pendingTranscript, sending, voice.utteranceReset], () => {
|
|
612
|
+
if (!disposed && live.value && !pushHolding.value && !changingCallMode.value &&
|
|
613
|
+
!pendingTranscript.value && !sending.value && !committing && microphoneMuted.value &&
|
|
614
|
+
recording?.finishAfterPending && voice.listening.value) {
|
|
615
|
+
// Retire intent before the asynchronous PCM drain so a receipt/reset
|
|
616
|
+
// cannot finalize this capture twice. Its original final watcher sends B.
|
|
617
|
+
delete recording.finishAfterPending;
|
|
618
|
+
void voice.stopListening().catch(cause => { error.value = cause.message; });
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
477
621
|
if (live.value && callMode.value === "hands-free" && !changingCallMode.value && !microphoneMuted.value && !capturing.value && !pendingTranscript.value && !sending.value && !voice.error.value) void startRecording(false, true);
|
|
478
622
|
});
|
|
479
623
|
async function startHeldRecording() { await startRecording(true); }
|
|
@@ -524,14 +668,18 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
524
668
|
text: binding.state.streamingReply?.text }), ({ messages = [], reply }) => {
|
|
525
669
|
if (!binding.id) return;
|
|
526
670
|
// Reconcile an uncertain send by identity, never by equal transcript text.
|
|
527
|
-
if (pendingTranscript.value && messages.some((message) => message.role === "user" && message.id === pendingTranscript.value.messageId)) {
|
|
671
|
+
if (pendingTranscript.value && messages.some((message) => message.role === "user" && message.id === pendingTranscript.value.messageId && message.receipt !== false)) {
|
|
528
672
|
dismissTranscript();
|
|
529
673
|
}
|
|
674
|
+
let invited = false;
|
|
530
675
|
for (const message of messages) {
|
|
676
|
+
// Unadmitted local records do not acknowledge a request or lift speech silence.
|
|
677
|
+
if (message.role === "user" && message.receipt === false) continue;
|
|
531
678
|
if (seen.has(message.id)) continue;
|
|
532
679
|
if (message.role === "user" && message.id === speechInvitation) {
|
|
533
680
|
speechSuppressed = false;
|
|
534
681
|
speechInvitation = "";
|
|
682
|
+
invited = true;
|
|
535
683
|
}
|
|
536
684
|
if (message.role === "assistant") queueReply(message, true);
|
|
537
685
|
seen.add(message.id);
|
|
@@ -546,7 +694,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
546
694
|
lastStreamingId = reply?.id || "";
|
|
547
695
|
if (reply?.text) {
|
|
548
696
|
queueReply(reply, reply.status === "completed");
|
|
549
|
-
if (!readAloud.value || !primed || reply.autonomous) seen.add(reply.id);
|
|
697
|
+
if (!readAloud.value || !primed || reply.autonomous || hiddenNarrationPage.value) seen.add(reply.id);
|
|
550
698
|
}
|
|
551
699
|
// Keep only the visible history plus active playback receipts in memory.
|
|
552
700
|
const retained = new Set([...messages.map((message) => message.id), reply?.id, ...speechQueue.map((item) => item.id),
|
|
@@ -554,24 +702,119 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
554
702
|
for (const id of seen) if (!retained.has(id)) seen.delete(id);
|
|
555
703
|
for (const id of replies.keys()) if (!retained.has(id)) replies.delete(id);
|
|
556
704
|
primed = true;
|
|
705
|
+
if (invited) refreshNarration();
|
|
557
706
|
void drainSpeech();
|
|
558
707
|
}, { immediate: true });
|
|
708
|
+
watch(() => {
|
|
709
|
+
const narration = binding.narration;
|
|
710
|
+
return {
|
|
711
|
+
id: binding.id, conversationId: binding.conversationId, narration,
|
|
712
|
+
loading: narration?.loading, eligible: narration?.eligible, working: narration?.working,
|
|
713
|
+
thinking: narration?.vocalizeThinking, interim: narration?.vocalizeInterimTurns,
|
|
714
|
+
sounds: narration?.thinkingSounds, turns: narration?.turns, enabled: narrationEnabled()
|
|
715
|
+
};
|
|
716
|
+
}, (current, previous) => {
|
|
717
|
+
clearNarrationTimers();
|
|
718
|
+
if (!current.narration || current.loading || current.id !== previous?.id || current.conversationId !== previous?.conversationId) {
|
|
719
|
+
narrationTracker.reset();
|
|
720
|
+
retireNarration();
|
|
721
|
+
if (!current.narration || current.loading) return;
|
|
722
|
+
}
|
|
723
|
+
if (!current.enabled) retireNarration();
|
|
724
|
+
else retireNarration(kind => kind === "thinking" ? !current.thinking
|
|
725
|
+
: kind === "commentary" ? !current.interim : !current.sounds || !current.working);
|
|
726
|
+
// Retire disabled tails before enabling a flag; settings are not a request to replay history.
|
|
727
|
+
if (previous && (current.enabled !== previous.enabled || current.thinking !== previous.thinking || current.interim !== previous.interim)) observeNarration(true, false);
|
|
728
|
+
refreshNarration(!current.enabled);
|
|
729
|
+
}, { immediate: true, deep: true });
|
|
559
730
|
watch(voice.error, (message) => { if (message) cancelSpeech("playback-error", "failed"); });
|
|
560
731
|
watch(() => error.value || voice.error.value, (message) => { if (message) binding.onError?.(message); });
|
|
561
732
|
watch([voice.activeSpeechTurnId, voice.canAppendSpeech], () => { void drainSpeech(); });
|
|
562
|
-
|
|
733
|
+
function currentTranscript() {
|
|
563
734
|
const pending = pendingTranscript.value;
|
|
564
|
-
const
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
}
|
|
735
|
+
const currentWords = recording && ["listening", "transcribing"].includes(voice.captureState.value)
|
|
736
|
+
&& voice.partialTranscript.value && recording.messageId !== pending?.messageId;
|
|
737
|
+
return currentWords ? { id: recording.messageId, text: voice.partialTranscript.value }
|
|
738
|
+
: pending ? { id: pending.messageId, text: pending.text } : null;
|
|
739
|
+
}
|
|
740
|
+
function canTakeTranscript(messageId) {
|
|
741
|
+
return !disposed && !starting.value && !sending.value && !committing && !takingTranscript.value
|
|
742
|
+
&& (pendingTranscript.value?.messageId === messageId || currentTranscript()?.id === messageId)
|
|
743
|
+
&& !binding.state.messages?.some(message => message.role === "user" && message.id === messageId && message.receipt !== false);
|
|
744
|
+
}
|
|
745
|
+
async function beginTranscriptEdit(messageId) {
|
|
746
|
+
if (!canTakeTranscript(messageId)) return false;
|
|
747
|
+
const pending = pendingTranscript.value;
|
|
748
|
+
if (pending && pending.messageId !== messageId) return false;
|
|
749
|
+
const selected = pending ? { id: pending.messageId, text: pending.text } : currentTranscript();
|
|
750
|
+
const capture = recording;
|
|
751
|
+
takingTranscript.value = true;
|
|
752
|
+
try {
|
|
753
|
+
// Retain the selected words and destination before cancelling capture.
|
|
754
|
+
// The existing pending-clear watcher resumes hands-free after resolution.
|
|
755
|
+
pendingTranscript.value = {
|
|
756
|
+
...(pending?.messageId === messageId ? pending : capture),
|
|
757
|
+
messageId, text: selected.text, reviewBeforeSend: true, editing: true
|
|
758
|
+
};
|
|
759
|
+
if (capture) {
|
|
760
|
+
if (capture.messageId !== messageId && live.value && callMode.value === "hands-free") {
|
|
761
|
+
// Newer continuous words retain their recording identity and endpoint.
|
|
762
|
+
if (!microphoneMuted.value) {
|
|
763
|
+
pendingTranscript.value.editPausedCaptureId = capture.messageId;
|
|
764
|
+
voice.setMicrophoneMuted(true);
|
|
765
|
+
}
|
|
766
|
+
} else if (pushHolding.value) await cancelPushToTalk();
|
|
767
|
+
else await cancelRecording();
|
|
768
|
+
}
|
|
769
|
+
return true;
|
|
770
|
+
} finally { takingTranscript.value = false; }
|
|
771
|
+
}
|
|
772
|
+
async function takeTranscript(messageId, transfer) {
|
|
773
|
+
if (!canTakeTranscript(messageId)) return false;
|
|
774
|
+
const pending = pendingTranscript.value;
|
|
775
|
+
const selected = pending?.messageId === messageId ? { id: messageId, text: pending.text } : currentTranscript();
|
|
776
|
+
takingTranscript.value = true;
|
|
777
|
+
try {
|
|
778
|
+
// The host rechecks its draft and transfers synchronously before cancellation.
|
|
779
|
+
if (transfer && transfer(selected.text) !== true) return false;
|
|
780
|
+
if (pendingTranscript.value?.messageId === messageId) dismissTranscript();
|
|
781
|
+
else if (recording?.messageId === messageId) {
|
|
782
|
+
if (pushHolding.value) await cancelPushToTalk();
|
|
783
|
+
else await cancelRecording();
|
|
784
|
+
if (!disposed && live.value && callMode.value === "hands-free" && !microphoneMuted.value) {
|
|
785
|
+
await startRecording(false, true);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
return true;
|
|
789
|
+
} finally { takingTranscript.value = false; }
|
|
790
|
+
}
|
|
791
|
+
function publishTranscript() {
|
|
792
|
+
const transcript = currentTranscript();
|
|
793
|
+
binding.onTranscript?.(transcript, { canTake: Boolean(transcript && canTakeTranscript(transcript.id)) });
|
|
794
|
+
}
|
|
795
|
+
watch([pendingTranscript, voice.captureState, voice.partialTranscript, voice.endpoint, sending, starting, takingTranscript], publishTranscript, { immediate: true });
|
|
569
796
|
watch(avatarVisual, (visual) => binding.onVisual?.(visual), { immediate: true });
|
|
570
797
|
const page = globalThis.window || globalThis;
|
|
798
|
+
const updateNarrationVisibility = () => {
|
|
799
|
+
narrationPageVisible.value = globalThis.document?.visibilityState !== "hidden";
|
|
800
|
+
if (disposed || !hiddenNarrationPage.value || !(recording || starting.value || committing || capturing.value)) return;
|
|
801
|
+
// This is the original document-hidden capture boundary, not avatar/body
|
|
802
|
+
// visibility. Retire B before cancellation; A's delivery stays untouched.
|
|
803
|
+
pushHolding.value = false;
|
|
804
|
+
pushInput = null;
|
|
805
|
+
if (!pendingTranscript.value) {
|
|
806
|
+
holding.value = false;
|
|
807
|
+
heldReview.value = false;
|
|
808
|
+
}
|
|
809
|
+
void discardRecording();
|
|
810
|
+
voice.stopSpeaking("page-hidden");
|
|
811
|
+
};
|
|
812
|
+
globalThis.document?.addEventListener?.("visibilitychange", updateNarrationVisibility);
|
|
571
813
|
const cancelStartup = () => { if (starting.value) void discardRecording(); else void cancelPushToTalk(); };
|
|
572
814
|
page.addEventListener?.("pagehide", cancelStartup);
|
|
573
815
|
page.addEventListener?.("blur", cancelPushToTalk);
|
|
574
816
|
onScopeDispose(() => {
|
|
817
|
+
globalThis.document?.removeEventListener?.("visibilitychange", updateNarrationVisibility);
|
|
575
818
|
page.removeEventListener?.("pagehide", cancelStartup);
|
|
576
819
|
page.removeEventListener?.("blur", cancelPushToTalk);
|
|
577
820
|
startupAbort?.abort(); startupAbort = null;
|
|
@@ -589,18 +832,21 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
589
832
|
stopSpeech();
|
|
590
833
|
await voice.close();
|
|
591
834
|
}
|
|
592
|
-
function editTranscript(text) {
|
|
593
|
-
if (pendingTranscript.value && !sending.value) pendingTranscript.value = { ...pendingTranscript.value, text: String(text) };
|
|
835
|
+
function editTranscript(text, messageId = pendingTranscript.value?.messageId) {
|
|
836
|
+
if (pendingTranscript.value && pendingTranscript.value.messageId === messageId && !sending.value) pendingTranscript.value = { ...pendingTranscript.value, text: String(text) };
|
|
594
837
|
}
|
|
838
|
+
const composerBlocked = computed(() => starting.value ||
|
|
839
|
+
(!live.value || callMode.value !== "hands-free") &&
|
|
840
|
+
(sending.value || Boolean(pendingTranscript.value) || capturing.value));
|
|
595
841
|
const hasUnsentSpeech = computed(() => capturing.value || starting.value || sending.value || Boolean(pendingTranscript.value));
|
|
596
842
|
return {
|
|
597
|
-
oneOffTalkMode, targetLabel, voice, readAloud, error, pendingTranscript, heldReview, holding, sending,
|
|
843
|
+
oneOffTalkMode, targetLabel, voice, readAloud, readAloudChangePending, error, pendingTranscript, heldReview, holding, sending,
|
|
598
844
|
live, starting, callMode, changingCallMode, pushHolding, microphoneMuted, capturing,
|
|
599
845
|
speechActive, talkLabel, liveLabel, callConnection, callStatus, callDetail, callModeBusy,
|
|
600
846
|
microphoneStatus, avatarVisual, callAudioLevel, status, heldTranscript, voiceWords, voiceAnswer,
|
|
601
847
|
stopSpeech, inviteSpeech, toggleReadAloud, enableSound, soundPreparing, deliverTranscript, cancelRecording,
|
|
602
848
|
discardRecording, dismissTranscript, talk, toggleMicrophoneMuted, changeCallMode,
|
|
603
849
|
startPushToTalk, finishPushToTalk, cancelPushToTalk, toggleLive, toggleHandsFree, startHeldRecording,
|
|
604
|
-
finishHeldRecording, discardHeldRecording, close, editTranscript, hasUnsentSpeech
|
|
850
|
+
finishHeldRecording, discardHeldRecording, close, editTranscript, canTakeTranscript, beginTranscriptEdit, takeTranscript, hasUnsentSpeech, composerBlocked
|
|
605
851
|
};
|
|
606
852
|
}
|