@jskit-ai/assistant-voice 0.1.13 → 0.1.15

package/README.md

@@ -149,6 +149,15 @@ replay history or automatically restart capture. If a service advertises several
 voices, the Voice settings cog offers Speaking voice for the next reply's speaker; an in-flight reply keeps
 its original speaker through all streamed phrases.
 
+Canonical replies retain the original bounded Markdown cleanup: inspect at most
+8,000 raw characters, then speak at most 4,000 normalized characters. A long code
+block inside that window does not consume the spoken allowance or hide the answer
+after it. Streaming keeps raw source offsets for revision checks; the existing
+queue counts normalized consumed prefixes and limits appended text. Reaching the
+speech limit ends synthesis, but a completed playback receipt still waits for
+the canonical final and actual audio drain. Optional activity narration keeps its
+separate original observed-delta limit.
+
 ## Server connection
 
 Register an authenticated WebSocket route with Fastify's WebSocket plugin:

package/examples/conversation/package.json

@@ -9,14 +9,14 @@
   },
   "dependencies": {
     "@fastify/websocket": "^11.2.0",
-    "@jskit-ai/assistant-core": "0.1.213",
-    "@jskit-ai/assistant-runtime": "0.1.206",
-    "@jskit-ai/assistant-voice": "0.1.13",
-    "@jskit-ai/connectors-catalog": "0.1.52",
-    "@jskit-ai/http-runtime": "0.1.233",
-    "@jskit-ai/kernel": "0.1.235",
-    "@jskit-ai/realtime": "0.1.232",
-    "@jskit-ai/shell-web": "0.1.239",
+    "@jskit-ai/assistant-core": "0.1.215",
+    "@jskit-ai/assistant-runtime": "0.1.208",
+    "@jskit-ai/assistant-voice": "0.1.15",
+    "@jskit-ai/connectors-catalog": "0.1.54",
+    "@jskit-ai/http-runtime": "0.1.235",
+    "@jskit-ai/kernel": "0.1.237",
+    "@jskit-ai/realtime": "0.1.234",
+    "@jskit-ai/shell-web": "0.1.241",
     "@mdi/js": "^7.4.47",
     "@tanstack/vue-query": "^5.90.5",
     "fastify": "^5.8.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jskit-ai/assistant-voice",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "type": "module",
   "description": "Duplex voice conversations with application-owned targets, authorization and speech services.",
   "scripts": {
@@ -18,7 +18,7 @@
     "examples"
   ],
   "dependencies": {
-    "@jskit-ai/assistant-core": "0.1.213",
+    "@jskit-ai/assistant-core": "0.1.215",
     "@mdi/js": "^7.4.47",
     "ws": "^8.20.1"
   },

package/src/client/voiceConversation.js

@@ -1,6 +1,6 @@
 import { computed, onScopeDispose, ref, watch } from "vue";
 import { useVoiceTransport } from "./voiceTransport.js";
-import { takeStreamingSpeech } from "../shared/protocol.js";
+import { speechTextFromAssistant, takeStreamingSpeech, VOICE_MAX_SPEECH_TEXT_CHARACTERS } from "../shared/protocol.js";
 import { createConversationNarrationTracker } from "./conversationNarration.js";
 
 /** Created inside the controller's effect scope, independent of a mounted screen. */
@@ -82,7 +82,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
     return "You can speak naturally, including while I reply.";
   });
   const callModeBusy = computed(() => starting.value || changingCallMode.value || pushHolding.value || sending.value || Boolean(pendingTranscript.value)
-    || Boolean(voice.partialTranscript.value.trim()) || capturing.value && !voice.listening.value);
+    || capturing.value && Boolean(voice.partialTranscript.value.trim()) || capturing.value && !voice.listening.value);
   const microphoneStatus = computed(() => microphoneMuted.value ? "Microphone muted" : voice.listening.value ? "Microphone on" : "Microphone off");
   const avatarVisual = computed(() => ({
     avatar: binding.avatar,
@@ -237,7 +237,13 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
         if (reply.started && !speechActive.value) { speechQueue.shift(); continue; }
         if (speechActive.value && !reply.started) return;
         if (reply.started && !voice.canAppendSpeech.value) return;
-        const chunk = takeStreamingSpeech(reply.raw.slice(reply.consumed), reply.final, reply.started ? 140 : 64);
+        const previousSpeech = reply.narrationKind ? "" : speechTextFromAssistant(reply.raw.slice(0, reply.consumed));
+        const maximumCharacters = reply.narrationKind ? (reply.started ? 140 : 64) : Math.max(1, Math.min(
+          reply.started ? 140 : 64,
+          VOICE_MAX_SPEECH_TEXT_CHARACTERS - previousSpeech.length,
+          VOICE_MAX_SPEECH_TEXT_CHARACTERS - reply.spokenCharacters
+        ));
+        const chunk = reply.speechCapped ? null : takeStreamingSpeech(reply.raw.slice(reply.consumed), reply.final, maximumCharacters);
         if (!chunk) {
           if (reply.final && reply.started) voice.endSpeech();
           else if (reply.final) {
@@ -248,11 +254,27 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
           return;
         }
         reply.consumed += chunk.consumed;
-        if (!chunk.text) continue;
+        let text = chunk.text;
+        if (!reply.narrationKind) {
+          // Keep offsets in raw Markdown. The original limit applies after its
+          // cleanup, including whitespace between separately streamed phrases.
+          const speech = speechTextFromAssistant(reply.raw.slice(0, reply.consumed));
+          const boundary = speech.startsWith(previousSpeech)
+            ? speech.slice(previousSpeech.length).match(/^\s*/u)[0].length : 0;
+          text = text.slice(0, Math.max(0, Math.min(
+            VOICE_MAX_SPEECH_TEXT_CHARACTERS - previousSpeech.length - boundary,
+            VOICE_MAX_SPEECH_TEXT_CHARACTERS - reply.spokenCharacters
+          )));
+          reply.spokenCharacters += text.length;
+          reply.speechCapped = speech.length >= VOICE_MAX_SPEECH_TEXT_CHARACTERS
+            || reply.spokenCharacters >= VOICE_MAX_SPEECH_TEXT_CHARACTERS;
+          reply.final ||= reply.speechCapped;
+        }
+        if (!text) continue;
         if (!reply.started) {
           reply.started = true;
           reply.turnId = crypto.randomUUID();
-          const accepted = await voice.speak(chunk.text, reply.turnId, { stream: true });
+          const accepted = await voice.speak(text, reply.turnId, { stream: true });
           if (disposed) return;
           if (reply.cancelled) {
             if (reply.narrationKind) continue;
@@ -262,8 +284,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
             cancelReply(reply, "no-audio");
             continue;
           }
-        } else if (!voice.appendSpeech(chunk.text)) return;
-        if (reply.final && reply.consumed >= reply.raw.length) { voice.endSpeech(); return; }
+        } else if (!voice.appendSpeech(text)) return;
+        if (reply.speechCapped || (reply.final && reply.consumed >= reply.raw.length)) { voice.endSpeech(); return; }
       }
     } catch (cause) { error.value = cause.message; stopSpeech(); }
     finally { draining = false; }
@@ -275,7 +297,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
       retireNarration();
       reply = {
         id: message.id, outputId: message.outputId || message.id, streamId: message.streamId,
-        raw: "", consumed: 0, final: false, canonicalFinal: false, started: false
+        raw: "", consumed: 0, spokenCharacters: 0, speechCapped: false,
+        final: false, canonicalFinal: false, started: false
       };
       replies.set(message.id, reply);
       speechQueue.push(reply);
@@ -289,7 +312,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
       return;
     }
     if (final) retireNarration();
-    const text = String(message.text || "").slice(0, 4000);
+    const text = String(message.text || "").slice(0, VOICE_MAX_SPEECH_TEXT_CHARACTERS * 2);
     const spokenPrefix = reply.raw.slice(0, reply.consumed);
     if (!text.startsWith(spokenPrefix) && !(final && text.trimEnd() === spokenPrefix.trimEnd())) {
       // A model revision cannot unsay words. Retire that projection; never splice
@@ -301,7 +324,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
     reply.consumed = Math.min(reply.consumed, text.length);
     reply.raw = text;
     reply.canonicalFinal ||= final;
-    reply.final = final || text.length >= 4000;
+    reply.final = final || reply.speechCapped || text.length >= VOICE_MAX_SPEECH_TEXT_CHARACTERS * 2;
     if (reply.canonicalFinal && reply.drained) emitReply(reply, "completed");
   }
   async function enableSound() {
@@ -649,6 +672,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
     const finished = recording;
     recording = null;
     if (voice.error.value) {
+      if (committing?.messageId === finished.messageId) committing = null;
       live.value = false;
       const text = voice.partialTranscript.value.trim() || voice.transcript.value.trim();
       if (text) {

package/src/client/voiceTransport.js

@@ -1,6 +1,6 @@
 import { computed, onScopeDispose, ref, watch } from "vue";
 
-import { speechTextFromAssistant } from "../shared/protocol.js";
+import { VOICE_MAX_SYNTHESIS_TIMEOUT_MS, speechTextFromAssistant } from "../shared/protocol.js";
 import { createMicrophoneCapture } from "./audioCapture.js";
 import {
   normalizeSpeechSegment,
@@ -63,6 +63,8 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
   let heartbeatTimer;
   let lastPongAt = 0;
   let lastSpeechProgressAt = 0;
+  let synthesisTimeoutMs = 20_000;
+  let lastStartedSegmentIndex = -1;
   let microphone = null;
   let preserveBufferedMicrophone = false;
   const openingMicrophones = new Set();
@@ -165,7 +167,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
       connectionLost();
       return;
     }
-    if (activeSpeechTurnId.value && synthesisPending.value && Date.now() - lastSpeechProgressAt > 20000) {
+    if (activeSpeechTurnId.value && synthesisPending.value && Date.now() - lastSpeechProgressAt > synthesisTimeoutMs) {
       connectionLost("Speech generation stalled. The answer remains in chat; use Talk to resume.");
       return;
     }
@@ -405,6 +407,8 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
     stopAfterSegment = Infinity;
     bufferedSeconds.value = 0;
     synthesisPending.value = 0;
+    synthesisTimeoutMs = 20_000;
+    lastStartedSegmentIndex = -1;
     receivingTurnId = "";
     receivingSegment = null;
     streamedSegments.clear();
@@ -477,11 +481,24 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
       return;
     }
     if (type === "speech.segment.start" && message.turnId === activeSpeechTurnId.value) {
+      if (!speechStreamEnded && Number.isSafeInteger(message.segmentIndex) &&
+          message.segmentIndex > lastStartedSegmentIndex) {
+        lastStartedSegmentIndex = message.segmentIndex;
+        synthesisTimeoutMs = 20_000;
+        if (Number.isSafeInteger(message.synthesisTimeoutMs) && message.synthesisTimeoutMs > 20_000 &&
+            message.synthesisTimeoutMs <= VOICE_MAX_SYNTHESIS_TIMEOUT_MS) {
+          // The engine acquired its queue slot. Allow its bounded cold job until
+          // first PCM; repeated metadata and pongs are not synthesis progress.
+          synthesisTimeoutMs = message.synthesisTimeoutMs;
+          lastSpeechProgressAt = Date.now();
+        }
+      }
       receivingSegment = { index: message.segmentIndex };
       streamedSegments.set(message.segmentIndex, receivingSegment);
       return;
     }
     if (type === "speech.chunk.end" && message.turnId === activeSpeechTurnId.value) {
+      synthesisTimeoutMs = 20_000;
       lastSpeechProgressAt = Date.now();
       synthesisPending.value = Math.max(0, synthesisPending.value - 1);
       return;
@@ -515,6 +532,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
   function handleSocketMessage(event) {
     if (event.data instanceof ArrayBuffer) {
       if (speechOutputEnabled.value && activeSpeechTurnId.value && receivingTurnId === activeSpeechTurnId.value && !speechStreamEnded) {
+        synthesisTimeoutMs = 20_000;
         lastSpeechProgressAt = Date.now();
         enqueuePcm(event.data);
       }

package/src/server/voiceDaemon.js

@@ -8,6 +8,7 @@ import {
   VOICE_MAX_AUDIO_FRAME_BYTES,
   VOICE_MAX_CONTROL_BYTES,
   VOICE_MAX_UTTERANCE_SECONDS,
+  VOICE_MAX_SYNTHESIS_TIMEOUT_MS,
   VOICE_PROTOCOL_VERSION,
   assertVoiceAudioFrame,
   parseVoiceClientControl,
@@ -356,7 +357,10 @@ function createVoiceDaemon({
                 sendJson(socket, "speech.start", { sampleRate: Number(engine.sampleRate), turnId });
               }
               const segmentIndex = active.segmentIndex++;
-              sendJson(socket, "speech.segment.start", { segmentIndex, turnId });
+              const synthesisTimeoutMs = engine.synthesisTimeoutMs;
+              sendJson(socket, "speech.segment.start", { segmentIndex, turnId,
+                ...(Number.isSafeInteger(synthesisTimeoutMs) && synthesisTimeoutMs > 20_000 &&
+                    synthesisTimeoutMs <= VOICE_MAX_SYNTHESIS_TIMEOUT_MS ? { synthesisTimeoutMs } : {}) });
               let sampleCount = 0;
               const result = text.trim() ? await engine.synthesize(text.trim(), {
                 onAudio(frame) {

package/src/shared/protocol.js

@@ -5,6 +5,7 @@ const VOICE_OUTPUT_SAMPLE_RATE = 22_050;
 const VOICE_MAX_CONTROL_BYTES = 16 * 1024;
 const VOICE_MAX_AUDIO_FRAME_BYTES = 64 * 1024;
 const VOICE_MAX_UTTERANCE_SECONDS = 60;
+const VOICE_MAX_SYNTHESIS_TIMEOUT_MS = 120_000;
 const VOICE_MAX_SPEECH_TEXT_CHARACTERS = 4_000;
 const VOICE_MAX_SPEECH_CHUNK_CHARACTERS = 280;
 const VOICE_SPEECH_EM_DASH_PAUSE_MS = 140;
@@ -421,6 +422,7 @@ export {
   VOICE_MAX_SPEECH_CHUNK_CHARACTERS,
   VOICE_MAX_SPEECH_TEXT_CHARACTERS,
   VOICE_MAX_UTTERANCE_SECONDS,
+  VOICE_MAX_SYNTHESIS_TIMEOUT_MS,
   VOICE_OUTPUT_SAMPLE_RATE,
   VOICE_PROTOCOL_VERSION,
   assertVoiceAudioFrame,