@jskit-ai/assistant-voice 0.1.2 → 0.1.3

package/examples/conversation/App.vue

@@ -2,12 +2,15 @@
 import { inject, nextTick, onScopeDispose, ref } from "vue";
 import { createVoiceConversationController, projectConversationVoiceState, VoiceConversationHost } from "@jskit-ai/assistant-voice/client";
 import { ShellErrorHost } from "@jskit-ai/shell-web/client";
+import { useAssistantConversationFactory } from "@jskit-ai/assistant-runtime/client";
 import Conversation from "./Conversation.vue";
 
 const example = inject("example.application");
 const closed = ref([]);
 const views = new Map();
 const error = ref("");
+const avatarSize = ref("compact");
+const acquireConversation = useAssistantConversationFactory();
 const controller = createVoiceConversationController({ connectSpeech: binding => binding.socketUrl });
 async function openText(id) {
   closed.value = closed.value.filter(value => value !== id);
@@ -18,12 +21,22 @@ function voiceBinding(runtime, label) {
   const identity = runtime.identity;
   let retained;
   return {
-    id: JSON.stringify([identity.actorKey, identity.conversationId]), label,
+    id: JSON.stringify([identity.actorKey, identity.conversationId]), conversationId: identity.conversationId, label,
+    get adapter() { return retained?.adapter.value; },
     socketUrl: `/api/conversations/${encodeURIComponent(identity.conversationId)}/voice`,
     get state() { return projectConversationVoiceState({ turns: runtime.turns.value, status: runtime.snapshot.value?.status }); },
     get available() { return runtime.available.value; },
     captureContext: () => ({ ...identity }),
-    retain() { retained = runtime.retain(); runtime = retained.runtime; },
+    retain() {
+      retained = acquireConversation({ conversationId: identity.conversationId, endpoint: identity.endpoint,
+        surfaceId: identity.targetSurfaceId, hostSurfaceId: identity.hostSurfaceId,
+        actorKey: identity.actorKey, workspaceSlug: identity.workspaceSlug,
+        presentation: { assistantLabel: label, layout: "compact" } });
+      if (retained.runtime.value !== runtime) {
+        retained.release(); retained = null;
+        throw new Error("Voice must retain the original conversation runtime.");
+      }
+    },
     release() { retained?.release(); retained = null; },
     submitText(text, { messageId, context } = {}) {
       if (context?.actorKey !== identity.actorKey || context?.conversationId !== identity.conversationId) {
@@ -78,7 +91,7 @@ onScopeDispose(() => { void controller.dispose(); });
         </section>
       </div>
     </main>
-    <VoiceConversationHost :controller="controller" />
+    <VoiceConversationHost v-model:avatar-size="avatarSize" :controller="controller" />
     <ShellErrorHost />
   </v-app>
 </template>

package/examples/conversation/package.json

@@ -9,14 +9,14 @@
   },
   "dependencies": {
     "@fastify/websocket": "^11.2.0",
-    "@jskit-ai/assistant-core": "0.1.202",
-    "@jskit-ai/assistant-runtime": "0.1.195",
-    "@jskit-ai/assistant-voice": "0.1.2",
-    "@jskit-ai/connectors-catalog": "0.1.41",
-    "@jskit-ai/http-runtime": "0.1.222",
-    "@jskit-ai/kernel": "0.1.224",
-    "@jskit-ai/realtime": "0.1.221",
-    "@jskit-ai/shell-web": "0.1.228",
+    "@jskit-ai/assistant-core": "0.1.203",
+    "@jskit-ai/assistant-runtime": "0.1.196",
+    "@jskit-ai/assistant-voice": "0.1.3",
+    "@jskit-ai/connectors-catalog": "0.1.42",
+    "@jskit-ai/http-runtime": "0.1.223",
+    "@jskit-ai/kernel": "0.1.225",
+    "@jskit-ai/realtime": "0.1.222",
+    "@jskit-ai/shell-web": "0.1.229",
     "@mdi/js": "^7.4.47",
     "@tanstack/vue-query": "^5.90.5",
     "fastify": "^5.8.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jskit-ai/assistant-voice",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "description": "Duplex voice conversations with application-owned targets, authorization and speech services.",
   "scripts": {
@@ -18,6 +18,7 @@
     "examples"
   ],
   "dependencies": {
+    "@jskit-ai/assistant-core": "0.1.203",
     "@mdi/js": "^7.4.47",
     "ws": "^8.20.1"
   },

package/src/client/VoiceConversation.vue

@@ -1,28 +1,18 @@
 <script setup>
-import { computed, ref, useId, watch } from "vue";
-import { mdiMicrophone, mdiMicrophoneOff, mdiStop, mdiVolumeHigh, mdiVolumeOff } from "@mdi/js";
+import { ref, watch } from "vue";
+import { AssistantConversationElement } from "@jskit-ai/assistant-core/client/conversation";
 import VoiceAvatar from "./VoiceAvatar.vue";
-import { useVoiceLauncher } from "./voiceLauncher.js";
+import VoiceConversationControls from "./VoiceConversationControls.vue";
 
-const props = defineProps({ session: { type: Object, required: true }, disabled: { type: Boolean, default: false } });
-const {
-  targetLabel, voice, error, microphoneMuted, live, starting, callMode,
-  callStatus, callAudioLevel, avatarVisual, speechActive, heldTranscript,
-  capturing, pendingTranscript, voiceWords, voiceAnswer, pushHolding, sending,
-  stopSpeech, startPushToTalk, finishPushToTalk, cancelPushToTalk, toggleHandsFree, readAloud, toggleReadAloud,
-  deliverTranscript, discardHeldRecording, editTranscript
-} = props.session;
-const handsFreeListening = computed(() => live.value && callMode.value === "hands-free" && !microphoneMuted.value);
-const canTalk = computed(() => !props.disabled && !sending.value && !pendingTranscript.value &&
-  (!capturing.value || voice.listening.value || pushHolding.value || starting.value));
-const talkLabel = computed(() => starting.value ? "Connecting…" : pushHolding.value ? "Release to send" : handsFreeListening.value ? "Pause" : "Talk");
-const gestureHint = useId();
-const gesture = useVoiceLauncher({
-  enabled: canTalk,
-  open: () => ({ startHeldRecording: startPushToTalk, finishHeldRecording: finishPushToTalk, discardHeldRecording: cancelPushToTalk }),
-  onTap: toggleHandsFree,
-  onError: cause => { error.value = cause.message; }
+const props = defineProps({
+  session: { type: Object, required: true }, disabled: { type: Boolean, default: false },
+  adapter: { type: Object, default: null },
+  avatarSize: { type: String, default: "compact" }
 });
+defineEmits(["update:avatarSize"]);
+const { targetLabel, voice, microphoneMuted, heldTranscript, pendingTranscript, sending,
+  avatarVisual, callAudioLevel, voiceWords, voiceAnswer } = props.session;
+const toolsTarget = ref(null);
 const voiceWordsTarget = ref(null);
 const voiceAnswerTarget = ref(null);
 const followWords = ref(true);
@@ -36,18 +26,23 @@ for (const [text, target, following] of [[voiceWords, voiceWordsTarget, followWo
 </script>
 
 <template>
-  <fieldset class="assistant-voice" :disabled="disabled">
-    <p v-if="error || voice.error.value" class="assistant-voice__error" role="alert">{{ error || voice.error.value }}</p>
+  <fieldset v-if="adapter" class="assistant-voice-conversation" :disabled="disabled">
+  <AssistantConversationElement :adapter="adapter" :avatar-size="avatarSize" @update:avatar-size="$emit('update:avatarSize', $event)">
+    <template #avatar="size"><slot name="avatar" v-bind="avatarVisual" :size="size.size" :height="size.height"><VoiceAvatar v-bind="avatarVisual" /></slot></template>
+    <template #composer-tools><div ref="toolsTarget" class="assistant-voice-conversation__tools" /></template>
+    <template #composer-feedback>
+      <VoiceConversationControls :session="session" :disabled="disabled" compact :tools-target="toolsTarget">
+        <template #work-control><slot name="work-control" /></template>
+      </VoiceConversationControls>
+    </template>
+  </AssistantConversationElement>
+  </fieldset>
+  <fieldset v-else class="assistant-voice" :disabled="disabled">
     <section class="assistant-voice__call" :aria-label="`${targetLabel} voice chat controls`">
       <div class="assistant-voice__call-body">
         <div class="assistant-voice__call-portrait" :style="{ '--call-audio-scale': 1 + callAudioLevel * .16 }" aria-hidden="true">
           <slot name="avatar" v-bind="avatarVisual"><VoiceAvatar v-bind="avatarVisual" /></slot>
         </div>
-        <div class="assistant-voice__call-status-row">
-          <p class="text-title-medium" role="status">{{ callStatus }}</p>
-          <v-btn v-if="speechActive" :icon="mdiStop" variant="text" min-width="48" min-height="48" :aria-label="`Stop ${targetLabel} speaking`" title="Stop speaking" @click="stopSpeech" />
-          <slot name="work-control" />
-        </div>
       </div>
       <div class="assistant-voice__captions" aria-label="Voice conversation text">
         <section ref="voiceWordsTarget" class="assistant-voice__caption" aria-label="Recognized words" tabindex="0" @scroll="followWords = isAtBottom($event.currentTarget)">
@@ -59,36 +54,14 @@ for (const [text, target, following] of [[voiceWords, voiceWordsTarget, followWo
           <p class="assistant-voice__caption-text text-body-medium">{{ voiceAnswer }}</p>
         </section>
       </div>
-      <section v-if="pendingTranscript && (pendingTranscript.reviewBeforeSend || !sending)" class="assistant-voice__review" aria-label="Review voice message">
-        <v-textarea :model-value="pendingTranscript.text" label="Review your message" :disabled="sending" rows="2" max-rows="3" auto-grow hide-details @update:model-value="editTranscript" />
-        <div class="assistant-voice__review-actions">
-          <v-btn :disabled="sending" min-height="48" @click="discardHeldRecording">Discard</v-btn>
-          <v-btn :disabled="sending || !pendingTranscript.text.trim()" min-height="48" color="primary" @click="deliverTranscript">{{ sending ? 'Sending…' : 'Send' }}</v-btn>
-        </div>
-      </section>
-      <div class="assistant-voice__talk-action">
-        <v-btn
-          class="assistant-voice__talk" color="primary" :variant="handsFreeListening ? 'tonal' : 'flat'" min-height="64" rounded="xl"
-          :prepend-icon="voice.listening.value && !microphoneMuted ? mdiMicrophone : mdiMicrophoneOff" :aria-pressed="handsFreeListening || pushHolding" :aria-describedby="gestureHint"
-          :disabled="!canTalk"
-          @pointerdown="gesture.pointerDown" @pointerup="gesture.pointerUp" @pointercancel="gesture.cancel" @lostpointercapture="gesture.cancel"
-          @keydown.space.prevent="gesture.keyDown" @keyup.space.prevent="gesture.keyUp"
-          @keydown.enter.prevent="gesture.keyDown" @keyup.enter.prevent="gesture.keyUp" @blur="gesture.cancel" @contextmenu.prevent @click="gesture.click"
-        >
-          {{ talkLabel }}
-        </v-btn>
-        <v-btn
-          :icon="readAloud ? mdiVolumeHigh : mdiVolumeOff" :aria-pressed="readAloud" variant="tonal" min-height="64" min-width="64" rounded="xl"
-          :aria-label="readAloud ? `Turn ${targetLabel} read-aloud off` : `Read ${targetLabel} answers aloud`"
-          :title="readAloud ? 'Spoken replies on' : 'Spoken replies off'" @click="toggleReadAloud"
-        />
-        <p :id="gestureHint" class="text-body-small text-medium-emphasis">{{ handsFreeListening ? 'Tap to pause · hold to speak' : 'Tap for hands-free · hold to speak' }}</p>
-      </div>
+      <VoiceConversationControls :session="session" :disabled="disabled"><template #work-control><slot name="work-control" /></template></VoiceConversationControls>
     </section>
   </fieldset>
 </template>
 
 <style scoped>
+.assistant-voice-conversation { display: flex; flex-direction: column; flex: 1 1 auto; min-height: 0; min-width: 0; height: 100%; border: 0; padding: 0; margin: 0; }
+.assistant-voice-conversation__tools { display: flex; flex-wrap: wrap; gap: 4px; }
 .assistant-voice { display: flex; flex-direction: column; flex: 1; min-height: 0; overflow: hidden; border: 0; padding: 0; margin: 0; min-width: 0; }
 .assistant-voice__call { width: 100%; flex: 1; min-height: 0; display: grid; grid-template-rows: minmax(0, 1fr) minmax(144px, 30%) auto; }
 .assistant-voice__call-body { min-height: 0; display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 16px 16px 0; text-align: center; }
@@ -101,17 +74,10 @@ for (const [text, target, following] of [[voiceWords, voiceWordsTarget, followWo
 .assistant-voice__caption { min-height: 0; padding: 8px 12px; border-radius: 12px; background: rgba(var(--v-theme-on-surface), .04); overflow-y: auto; overscroll-behavior: contain; scrollbar-gutter: stable; touch-action: pan-y; }
 .assistant-voice__caption > strong { display: block; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
 .assistant-voice__caption-text { margin: 0; white-space: pre-wrap; overflow-wrap: anywhere; }
-.assistant-voice__talk-action { justify-self: center; width: min(288px, 100%); display: grid; grid-template-columns: minmax(0, 1fr) 64px; gap: 6px 8px; padding: 16px; }
-.assistant-voice__talk { touch-action: none; user-select: none; -webkit-touch-callout: none; }
-.assistant-voice__talk-action p { grid-column: 1 / -1; margin: 0; text-align: center; white-space: nowrap; }
-.assistant-voice__review { flex: 0 0 auto; max-height: 160px; overflow-y: auto; padding: 8px 16px; }
-.assistant-voice__review-actions { display: flex; justify-content: flex-end; gap: 8px; }
-.assistant-voice__error { padding: 8px 16px; color: rgb(var(--v-theme-error)); }
 @media (max-height: 650px) {
   .assistant-voice__call-body { padding-top: 0; }
   .assistant-voice__captions { gap: 4px; }
   .assistant-voice__caption { padding-block: 4px; }
-  .assistant-voice__talk-action { padding-top: 8px; }
 }
 @media (prefers-reduced-motion: reduce) { .assistant-voice__call-portrait::before { transition: none; transform: none; } }
 </style>

package/src/client/VoiceConversationControls.vue

@@ -0,0 +1,85 @@
+<script setup>
+import { computed, useId } from "vue";
+import { mdiMicrophone, mdiMicrophoneOff, mdiStop, mdiVolumeHigh, mdiVolumeOff } from "@mdi/js";
+import { useVoiceLauncher } from "./voiceLauncher.js";
+
+const props = defineProps({ session: { type: Object, required: true }, disabled: { type: Boolean, default: false }, compact: { type: Boolean, default: false }, toolsTarget: { type: Object, default: null } });
+const {
+  targetLabel, voice, error, microphoneMuted, live, starting, callMode,
+  callStatus, speechActive, heldTranscript,
+  capturing, pendingTranscript, pushHolding, sending,
+  stopSpeech, startPushToTalk, finishPushToTalk, cancelPushToTalk, toggleHandsFree, readAloud, toggleReadAloud,
+  deliverTranscript, discardHeldRecording, editTranscript
+} = props.session;
+const handsFreeListening = computed(() => live.value && callMode.value === "hands-free" && !microphoneMuted.value);
+const canTalk = computed(() => !props.disabled && !sending.value && !pendingTranscript.value &&
+  (!capturing.value || voice.listening.value || pushHolding.value || starting.value));
+const talkLabel = computed(() => starting.value ? "Connecting…" : pushHolding.value ? "Release to send" : handsFreeListening.value ? "Pause" : "Talk");
+const gestureHint = useId();
+const gesture = useVoiceLauncher({
+  enabled: canTalk,
+  open: () => ({ startHeldRecording: startPushToTalk, finishHeldRecording: finishPushToTalk, discardHeldRecording: cancelPushToTalk }),
+  onTap: toggleHandsFree,
+  onError: cause => { error.value = cause.message; }
+});
+</script>
+
+<template>
+  <fieldset class="assistant-voice-controls" :class="{ 'assistant-voice-controls--compact': compact }" :disabled="disabled">
+    <p v-if="error || voice.error.value" class="assistant-voice__error" role="alert">{{ error || voice.error.value }}</p>
+    <p v-if="callStatus" class="assistant-voice-controls__status text-body-small" role="status">{{ callStatus }}</p>
+    <p v-if="compact && heldTranscript && !pendingTranscript" class="assistant-voice-controls__provisional" role="status" aria-label="Recognized words">Not sent · {{ heldTranscript }}</p>
+      <section v-if="pendingTranscript && (pendingTranscript.reviewBeforeSend || !sending)" class="assistant-voice__review" aria-label="Review voice message">
+        <v-textarea :model-value="pendingTranscript.text" label="Review your message" :disabled="sending" :density="compact ? 'compact' : 'default'" :rows="compact ? 1 : 2" :max-rows="compact ? 2 : 3" auto-grow hide-details @update:model-value="editTranscript" />
+        <div class="assistant-voice__review-actions">
+          <v-btn :disabled="sending" min-height="48" @click="discardHeldRecording">Discard</v-btn>
+          <v-btn :disabled="sending || !pendingTranscript.text.trim()" min-height="48" color="primary" @click="deliverTranscript">{{ sending ? 'Sending…' : 'Send' }}</v-btn>
+        </div>
+      </section>
+    <Teleport :to="toolsTarget" :disabled="!toolsTarget">
+      <fieldset class="assistant-voice-controls__tools" :class="{ 'assistant-voice-controls__tools--compact': compact }" :disabled="disabled" :aria-label="`${targetLabel} voice chat controls`">
+          <v-btn v-if="speechActive" :icon="mdiStop" variant="text" min-width="48" min-height="48" :aria-label="`Stop ${targetLabel} speaking`" title="Stop speaking" @click="stopSpeech" />
+          <slot name="work-control" />
+      <div class="assistant-voice__talk-action">
+        <v-btn
+          class="assistant-voice__talk" color="primary" :variant="handsFreeListening ? 'tonal' : 'flat'" :min-height="compact ? 48 : 64" rounded="xl"
+          :prepend-icon="voice.listening.value && !microphoneMuted ? mdiMicrophone : mdiMicrophoneOff" :aria-pressed="handsFreeListening || pushHolding" :aria-describedby="gestureHint"
+          :disabled="!canTalk"
+          @pointerdown="gesture.pointerDown" @pointerup="gesture.pointerUp" @pointercancel="gesture.cancel" @lostpointercapture="gesture.cancel"
+          @keydown.space.prevent="gesture.keyDown" @keyup.space.prevent="gesture.keyUp"
+          @keydown.enter.prevent="gesture.keyDown" @keyup.enter.prevent="gesture.keyUp" @blur="gesture.cancel" @contextmenu.prevent @click="gesture.click"
+        >
+          {{ talkLabel }}
+        </v-btn>
+        <v-btn
+          :icon="readAloud ? mdiVolumeHigh : mdiVolumeOff" :aria-pressed="readAloud" variant="tonal" :min-height="compact ? 48 : 64" :min-width="compact ? 48 : 64" rounded="xl"
+          :aria-label="readAloud ? `Turn ${targetLabel} read-aloud off` : `Read ${targetLabel} answers aloud`"
+          :title="readAloud ? 'Spoken replies on' : 'Spoken replies off'" @click="toggleReadAloud"
+        />
+        <p :id="gestureHint" class="text-body-small text-medium-emphasis">{{ handsFreeListening ? 'Tap to pause · hold to speak' : 'Tap for hands-free · hold to speak' }}</p>
+      </div>
+      </fieldset>
+    </Teleport>
+  </fieldset>
+</template>
+
+<style scoped>
+.assistant-voice-controls, .assistant-voice-controls__tools { border: 0; padding: 0; margin: 0; min-width: 0; }
+.assistant-voice-controls { flex: 0 0 auto; }
+.assistant-voice-controls--compact { flex: 1 1 100%; width: 100%; }
+.assistant-voice-controls__tools { display: flex; align-items: center; justify-content: center; flex-wrap: wrap; }
+.assistant-voice-controls__status { text-align: center; margin: 0; }
+.assistant-voice-controls__provisional { margin: 0; max-height: 3rem; overflow-y: auto; white-space: pre-wrap; overflow-wrap: anywhere; }
+.assistant-voice__talk-action { width: min(288px, 100%); display: grid; grid-template-columns: minmax(0, 1fr) 64px; gap: 6px 8px; padding: 16px; }
+.assistant-voice__talk { touch-action: none; user-select: none; -webkit-touch-callout: none; }
+.assistant-voice__talk-action p { grid-column: 1 / -1; margin: 0; text-align: center; white-space: nowrap; }
+.assistant-voice__review { flex: 0 0 auto; max-height: 160px; overflow-y: auto; padding: 8px 16px; }
+.assistant-voice__review-actions { display: flex; justify-content: flex-end; gap: 8px; }
+.assistant-voice__error { padding: 8px 16px; color: rgb(var(--v-theme-error)); }
+.assistant-voice-controls__tools--compact { justify-content: flex-start; gap: 4px; }
+.assistant-voice-controls__tools--compact .assistant-voice__talk-action { display: flex; padding: 0; width: auto; gap: 4px; }
+.assistant-voice-controls__tools--compact .assistant-voice__talk-action p { position: absolute; width: 1px; height: 1px; padding: 0; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
+.assistant-voice-controls--compact .assistant-voice__review { max-height: 128px; padding: 0; }
+.assistant-voice-controls--compact .assistant-voice__error { margin: 0; padding: 0; }
+@media (max-height: 650px) { .assistant-voice__talk-action { padding-top: 8px; } }
+</style>

package/src/client/VoiceConversationHost.vue

@@ -3,7 +3,8 @@ import { mdiCog, mdiHeadset, mdiStop } from "@mdi/js";
 import ConversationDialog from "./ConversationDialog.vue";
 import VoiceConversation from "./VoiceConversation.vue";
 
-const props = defineProps({ controller: { type: Object, required: true }, activator: { type: Object, default: null } });
+const props = defineProps({ controller: { type: Object, required: true }, activator: { type: Object, default: null }, avatarSize: { type: String, default: "compact" } });
+defineEmits(["update:avatarSize"]);
 const state = props.controller.state;
 async function invoke(operation) {
   try { await operation(); }
@@ -53,12 +54,14 @@ async function openText() {
       </div>
     </v-alert>
     <div class="voice-host__body">
-      <VoiceConversation :key="state.binding.id" :session="state.session" :disabled="state.busy">
+      <slot name="conversation" :binding="state.binding" :session="state.session" :disabled="state.busy">
+      <VoiceConversation :key="state.binding.id" :session="state.session" :disabled="state.busy" :adapter="state.binding.adapter || null" :avatar-size="avatarSize" @update:avatar-size="$emit('update:avatarSize', $event)">
         <template v-if="$slots.avatar" #avatar="visual"><slot name="avatar" v-bind="visual" :binding="state.binding" /></template>
         <template #work-control>
           <v-btn v-if="state.binding.cancelWork && state.binding.state.status === 'working'" :disabled="state.busy" :icon="mdiStop" variant="text" aria-label="Stop agent work" title="Stop agent work" @click="invoke(state.binding.cancelWork)" />
         </template>
       </VoiceConversation>
+      </slot>
     </div>
   </ConversationDialog>
   <template v-if="state.session && !state.visible">

package/src/client/index.js

@@ -3,6 +3,7 @@ export { createVoiceConversationController } from "./voiceController.js";
 export { default as VoiceConversationHost } from "./VoiceConversationHost.vue";
 export { default as ConversationDialog } from "./ConversationDialog.vue";
 export { default as VoiceConversation } from "./VoiceConversation.vue";
+export { default as VoiceConversationControls } from "./VoiceConversationControls.vue";
 
 export { useVoiceLauncher } from "./voiceLauncher.js";
 export { useVoiceAvatar } from "./voiceAvatar.js";