@jskit-ai/assistant-voice 0.0.0-stage → 0.1.2

package/src/client/audioCapture.js

@@ -0,0 +1,198 @@
+const CAPTURE_WORKLET_NAME = "jskit-voice-capture";
+const CAPTURE_FLUSH_TIMEOUT_MS = 500;
+const CAPTURE_FRAME_SAMPLES = 1_024;
+const CAPTURE_WORKLET_SOURCE = `
+class AssistantVoiceCapture extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.accepting = true;
+    this.muted = false;
+    this.frames = [];
+    this.length = 0;
+    this.port.onmessage = (event) => {
+      if (event.data?.type === "mute") {
+        this.muted = event.data.muted;
+        this.frames = [];
+        this.length = 0;
+        return;
+      }
+      if (event.data?.type !== "flush") return;
+      this.accepting = false;
+      this.emit();
+      this.port.postMessage({ type: "flushed" });
+    };
+  }
+
+  emit() {
+    if (!this.length) return;
+    const samples = new Float32Array(this.length);
+    let offset = 0;
+    for (const frame of this.frames) {
+      samples.set(frame, offset);
+      offset += frame.length;
+    }
+    this.frames = [];
+    this.length = 0;
+    this.port.postMessage(samples, [samples.buffer]);
+  }
+
+  process(inputs) {
+    if (!this.accepting || this.muted) return true;
+    const input = inputs[0]?.[0];
+    if (!input?.length) return true;
+    this.frames.push(Float32Array.from(input));
+    this.length += input.length;
+    if (this.length >= ${CAPTURE_FRAME_SAMPLES}) this.emit();
+    return true;
+  }
+}
+registerProcessor("${CAPTURE_WORKLET_NAME}", AssistantVoiceCapture);
+`;
+
+function float32ToPcm16ArrayBuffer(samples) {
+  const output = new ArrayBuffer(samples.length * 2);
+  const view = new DataView(output);
+  for (let index = 0; index < samples.length; index += 1) {
+    const sample = Math.max(-1, Math.min(1, Number(samples[index]) || 0));
+    view.setInt16(index * 2, sample < 0 ? sample * 32768 : sample * 32767, true);
+  }
+  return output;
+}
+
+async function createMicrophoneCapture({ onLevel, onPcm, targetSampleRate = 16_000 } = {}) {
+  const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
+  if (!AudioContextCtor) {
+    throw new Error("This browser does not support microphone audio processing.");
+  }
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {
+      autoGainControl: true,
+      channelCount: 1,
+      echoCancellation: true,
+      noiseSuppression: true,
+      sampleRate: { ideal: targetSampleRate }
+    },
+    video: false
+  });
+  const stopTracks = () => {
+    for (const track of stream.getTracks()) {
+      track.stop();
+    }
+  };
+  let context = null;
+  try {
+    context = new AudioContextCtor({
+      latencyHint: "interactive",
+      sampleRate: targetSampleRate
+    });
+    await context.resume();
+    if (Number(context.sampleRate) !== targetSampleRate) {
+      throw new Error(`This browser could not capture ${targetSampleRate} Hz microphone audio.`);
+    }
+    const workletUrl = URL.createObjectURL(new Blob([CAPTURE_WORKLET_SOURCE], {
+      type: "text/javascript"
+    }));
+    try {
+      await context.audioWorklet.addModule(workletUrl);
+    } finally {
+      URL.revokeObjectURL(workletUrl);
+    }
+    const source = context.createMediaStreamSource(stream);
+    const capture = new AudioWorkletNode(context, CAPTURE_WORKLET_NAME, {
+      channelCount: 1,
+      numberOfInputs: 1,
+      numberOfOutputs: 1,
+      outputChannelCount: [1]
+    });
+    const silence = context.createGain();
+    silence.gain.value = 0;
+    let finishFlush = null;
+    let muted = false;
+    capture.port.onmessage = (event) => {
+      if (event.data?.type === "flushed") {
+        finishFlush?.();
+        return;
+      }
+      if (muted) return;
+      const samples = event.data instanceof Float32Array
+        ? event.data
+        : new Float32Array(event.data);
+      let squaredTotal = 0;
+      for (const sample of samples) {
+        squaredTotal += sample * sample;
+      }
+      onLevel?.(samples.length
+        ? Math.min(1, Math.sqrt(squaredTotal / samples.length) * 7)
+        : 0);
+      onPcm?.(float32ToPcm16ArrayBuffer(samples));
+    };
+    source.connect(capture);
+    capture.connect(silence);
+    silence.connect(context.destination);
+
+    let closePromise = null;
+    const flush = () => new Promise((resolve) => {
+      let finished = false;
+      const timeout = globalThis.setTimeout(() => {
+        if (!finished) {
+          finished = true;
+          finishFlush = null;
+          resolve();
+        }
+      }, CAPTURE_FLUSH_TIMEOUT_MS);
+      finishFlush = () => {
+        if (finished) return;
+        finished = true;
+        globalThis.clearTimeout(timeout);
+        finishFlush = null;
+        resolve();
+      };
+      try {
+        capture.port.postMessage({ type: "flush" });
+      } catch {
+        finishFlush();
+      }
+    });
+    return Object.freeze({
+      setMuted(value) {
+        if (muted === Boolean(value)) return;
+        muted = Boolean(value);
+        for (const track of stream.getTracks()) track.enabled = !muted;
+        capture.port.postMessage({ type: "mute", muted });
+        if (muted) onLevel?.(0);
+      },
+      close() {
+        if (!closePromise) {
+          closePromise = (async () => {
+            try {
+              await flush();
+            } finally {
+              onLevel?.(0);
+              capture.port.onmessage = null;
+              try {
+                source.disconnect();
+                capture.disconnect();
+                silence.disconnect();
+              } finally {
+                stopTracks();
+                await context.close();
+              }
+            }
+          })();
+        }
+        return closePromise;
+      }
+    });
+  } catch (error) {
+    stopTracks();
+    if (context && context.state !== "closed") {
+      await context.close().catch(() => null);
+    }
+    throw error;
+  }
+}
+
+export {
+  createMicrophoneCapture,
+  float32ToPcm16ArrayBuffer
+};

package/src/client/conversationNarration.js

@@ -0,0 +1,108 @@
+import {
+  VOICE_MAX_SPEECH_TEXT_CHARACTERS,
+  speechPauseDurationMs,
+  speechPhrases,
+  speechTextFromAssistant,
+  splitSpeechText
+} from "../shared/protocol.js";
+
+const ACTIVITY_SPEECH_CHARACTERS = 300;
+
+function latestConversationFinalNarrationEntry(turns = []) {
+  const conversationTurns = Array.isArray(turns) ? turns : [];
+  for (let turnIndex = conversationTurns.length - 1; turnIndex >= 0; turnIndex -= 1) {
+    const turn = conversationTurns[turnIndex];
+    const turnId = String(turn?.turnId || turnIndex + 1).trim();
+    const finalText = String(turn?.assistant?.text || "").trim();
+    if (finalText && turn.assistant.status !== "inProgress") {
+      return {
+        key: `final:${turnId}`,
+        kind: "final",
+        text: finalText
+      };
+    }
+  }
+  return null;
+}
+
+function createConversationNarrationTracker() {
+  let primed = false;
+  let activityTurnId = "";
+  const finalKeys = new Set();
+  const activityText = new Map();
+
+  function reset() {
+    primed = false;
+    activityTurnId = "";
+    finalKeys.clear();
+    activityText.clear();
+  }
+
+  function observe(turns = [], {
+    includeFinals = false,
+    settled = false,
+    enabled = true,
+    vocalizeThinking = false,
+    vocalizeInterimTurns = false
+  } = {}) {
+    const conversationTurns = Array.isArray(turns) ? turns : [];
+    const turn = conversationTurns.at(-1);
+    const turnId = String(turn?.turnId || conversationTurns.length).trim();
+    if (turnId !== activityTurnId) {
+      activityTurnId = turnId;
+      activityText.clear();
+    }
+    const activity = (Array.isArray(turn?.messages) ? turn.messages : [])
+      .filter((message) => ["thinking", "commentary"].includes(message?.role));
+    if (!activity.length) {
+      activity.push(...[
+        ...(Array.isArray(turn?.thinking) ? turn.thinking : []),
+        ...(Array.isArray(turn?.commentary) ? turn.commentary : [])
+      ].sort((left, right) => String(left?.at || "").localeCompare(String(right?.at || ""))));
+    }
+    const entries = [];
+    for (const [index, message] of activity.entries()) {
+      const kind = message?.role;
+      if (!["thinking", "commentary"].includes(kind) || message.status === "inProgress") continue;
+      const key = `${turnId}:${kind}:${message.messageId || index}`;
+      const rawText = String(message.text || "");
+      // Track the whole message; only individual outgoing utterances are bounded.
+      const phrases = speechPhrases(speechTextFromAssistant(rawText,
+        Math.max(rawText.length, VOICE_MAX_SPEECH_TEXT_CHARACTERS)));
+      const fullText = phrases.join(" ");
+      const complete = settled || index < activity.length - 1 || Boolean(turn?.assistant);
+      if (!complete && !speechPauseDurationMs(phrases.at(-1))) phrases.pop();
+      const text = phrases.join(" ");
+      const previous = activityText.get(key) || "";
+      // A later stream observation must not retire a tail already flushed after settling.
+      if (text.length < previous.length && fullText.startsWith(previous)) continue;
+      activityText.set(key, text);
+      const selected = kind === "thinking" ? vocalizeThinking : vocalizeInterimTurns;
+      // Observe disabled/old content too, so enabling speech never replays it.
+      // A rewritten prefix is also retired instead of speaking it twice.
+      if (!primed || !enabled || !selected || !text.startsWith(previous)) continue;
+      for (const chunk of splitSpeechText(text.slice(previous.length), ACTIVITY_SPEECH_CHARACTERS)) {
+        entries.push({ kind, text: chunk });
+      }
+    }
+
+    const final = latestConversationFinalNarrationEntry(conversationTurns);
+    if (!primed) {
+      if (final) finalKeys.add(final.key);
+      primed = true;
+      return [];
+    }
+    if (includeFinals && final && !finalKeys.has(final.key)) {
+      finalKeys.add(final.key);
+      return enabled ? [final] : [];
+    }
+    return entries;
+  }
+
+  return { observe, reset };
+}
+
+export {
+  createConversationNarrationTracker,
+  latestConversationFinalNarrationEntry
+};

package/src/client/conversationVoiceState.js

@@ -0,0 +1,18 @@
+/** Canonical history is authoritative; only user and assistant output is spoken. */
+export function projectConversationVoiceState({ turns = [], status = "ready", interimReply = null } = {}) {
+  const messages = [];
+  let streamingReply = null;
+  for (const turn of turns) {
+    if (turn.user) messages.push({ ...turn.user, role: "user", id: turn.user.messageId || `${turn.turnId}:user` });
+    if (turn.assistant?.text) {
+      const reply = { ...turn.assistant, role: "assistant", id: turn.assistant.outputId || `${turn.turnId}:assistant` };
+      if (turn.pending || ["starting", "inProgress"].includes(reply.status)) {
+        // Production notifications kept live output separate until their saved reply.
+        if (reply.origin === "application") Object.assign(reply, { id: `${reply.id}:stream`, autonomous: true });
+        streamingReply = reply;
+      }
+      else messages.push(reply);
+    }
+  }
+  return { messages, streamingReply: interimReply ? { ...interimReply, id: interimReply.outputId || interimReply.id } : streamingReply, status };
+}

package/src/client/index.js

@@ -0,0 +1,9 @@
+export { projectConversationVoiceState } from "./conversationVoiceState.js";
+export { createVoiceConversationController } from "./voiceController.js";
+export { default as VoiceConversationHost } from "./VoiceConversationHost.vue";
+export { default as ConversationDialog } from "./ConversationDialog.vue";
+export { default as VoiceConversation } from "./VoiceConversation.vue";
+
+export { useVoiceLauncher } from "./voiceLauncher.js";
+export { useVoiceAvatar } from "./voiceAvatar.js";
+export { default as VoiceAvatar } from "./VoiceAvatar.vue";

package/src/client/voiceAvatar.js

@@ -0,0 +1,234 @@
+import { computed, onBeforeUnmount, onMounted, ref, watch } from "vue";
+
+const IDLE_GESTURES = Object.freeze([
+  "glance-left",
+  "tiny-nod",
+  "glance-right",
+  "brow-lift",
+  "glance-up",
+  "curious-tilt",
+  "smile-brighten",
+  "friendly-lean",
+  "shoulder-lift",
+  "earring-sway"
+]);
+const IDLE_MOUTH_POSES = Object.freeze({
+  "tiny-nod": "warm-smile",
+  "curious-tilt": "warm-smile",
+  "smile-brighten": "smile",
+  "friendly-lean": "warm-smile"
+});
+const IDLE_GESTURE_DURATION_MS = 1_100;
+const LISTENING_EXPRESSIONS = Object.freeze([
+  "attentive",
+  "tiny-nod",
+  "follow-left",
+  "tilt-left",
+  "follow-right",
+  "tilt-right"
+]);
+const LISTENING_MOUTH_POSES = Object.freeze({
+  "attentive": "soft-smile",
+  "tiny-nod": "soft-smile"
+});
+const PRE_THINKING_EXPRESSIONS = Object.freeze([
+  "settle",
+  "glance-left",
+  "center",
+  "glance-right",
+  "tiny-tilt",
+  "ready"
+]);
+const PRE_THINKING_MOUTH_POSES = Object.freeze({
+  "ready": "soft-smile",
+  "tiny-tilt": "thoughtful"
+});
+const THINKING_EXPRESSIONS = Object.freeze([
+  "look-up-left",
+  "focus",
+  "look-up-right",
+  "weigh-left",
+  "weigh-right",
+  "idea"
+]);
+const THINKING_MOUTH_POSES = Object.freeze({
+  "idea": "soft-smile",
+  "weigh-left": "closed",
+  "weigh-right": "closed"
+});
+const ACTIVE_EXPRESSION_DURATION_MS = Object.freeze({
+  listening: 1_350,
+  "pre-thinking": 1_450,
+  thinking: 1_700
+});
+const ACTIVE_EXPRESSION_SEQUENCES = Object.freeze({
+  listening: LISTENING_EXPRESSIONS,
+  "pre-thinking": PRE_THINKING_EXPRESSIONS,
+  thinking: THINKING_EXPRESSIONS
+});
+
+export function useVoiceAvatar(props) {
+  const blinking = ref(false);
+  const activeExpression = ref("");
+  const idleGesture = ref("");
+  const reducedMotion = ref(false);
+  let blinkTimer = 0;
+  let activeExpressionTimer = 0;
+  const activeExpressionIndexes = {
+    listening: Math.floor(Math.random() * LISTENING_EXPRESSIONS.length),
+    "pre-thinking": Math.floor(Math.random() * PRE_THINKING_EXPRESSIONS.length),
+    thinking: Math.floor(Math.random() * THINKING_EXPRESSIONS.length)
+  };
+  let idleGestureIndex = Math.floor(Math.random() * IDLE_GESTURES.length);
+  let idleGestureTimer = 0;
+  let reopenTimer = 0;
+  let reducedMotionQuery = null;
+
+  const expression = computed(() => String(props.state || "idle"));
+  const visibleMouthPose = computed(() => {
+    if (expression.value === "idle") {
+      return IDLE_MOUTH_POSES[idleGesture.value] || "soft-smile";
+    }
+    if (expression.value === "thinking") {
+      return THINKING_MOUTH_POSES[activeExpression.value] || "thoughtful";
+    }
+    if (expression.value === "pre-thinking") {
+      return PRE_THINKING_MOUTH_POSES[activeExpression.value] || "closed";
+    }
+    if (expression.value === "listening") {
+      return LISTENING_MOUTH_POSES[activeExpression.value] || "closed";
+    }
+    if (expression.value !== "speaking") {
+      return "closed";
+    }
+    const pose = String(props.mouthPose || "closed");
+    if (props.mouthLevel < 0.045 || !["open", "round", "small", "wide"].includes(pose)) {
+      return "closed";
+    }
+    return pose;
+  });
+
+  function clearBlinkTimers() {
+    window.clearTimeout(blinkTimer);
+    window.clearTimeout(reopenTimer);
+    blinkTimer = 0;
+    reopenTimer = 0;
+  }
+
+  function clearIdleGesture() {
+    window.clearTimeout(idleGestureTimer);
+    idleGestureTimer = 0;
+    idleGesture.value = "";
+  }
+
+  function clearActiveExpression() {
+    window.clearTimeout(activeExpressionTimer);
+    activeExpressionTimer = 0;
+    activeExpression.value = "";
+  }
+
+  function scheduleBlink(delay = 2_800 + (Math.random() * 3_400)) {
+    window.clearTimeout(blinkTimer);
+    if (reducedMotion.value) {
+      return;
+    }
+    blinkTimer = window.setTimeout(() => {
+      const doubleBlink = Math.random() < 0.16;
+      blinking.value = true;
+      reopenTimer = window.setTimeout(() => {
+        blinking.value = false;
+        if (!doubleBlink) {
+          scheduleBlink();
+          return;
+        }
+        blinkTimer = window.setTimeout(() => {
+          blinking.value = true;
+          reopenTimer = window.setTimeout(() => {
+            blinking.value = false;
+            scheduleBlink();
+          }, 115);
+        }, 155);
+      }, 135);
+    }, delay);
+  }
+
+  function scheduleIdleGesture(delay = 2_400 + (Math.random() * 2_800)) {
+    window.clearTimeout(idleGestureTimer);
+    if (reducedMotion.value || expression.value !== "idle") {
+      return;
+    }
+    idleGestureTimer = window.setTimeout(() => {
+      idleGesture.value = IDLE_GESTURES[idleGestureIndex];
+      idleGestureIndex = (idleGestureIndex + 1) % IDLE_GESTURES.length;
+      idleGestureTimer = window.setTimeout(() => {
+        idleGesture.value = "";
+        scheduleIdleGesture();
+      }, IDLE_GESTURE_DURATION_MS);
+    }, delay);
+  }
+
+  function showNextActiveExpression() {
+    const state = expression.value;
+    const sequence = ACTIVE_EXPRESSION_SEQUENCES[state] || [];
+    if (!sequence.length) {
+      clearActiveExpression();
+      return;
+    }
+    const index = activeExpressionIndexes[state] || 0;
+    activeExpression.value = sequence[index];
+    activeExpressionIndexes[state] = (index + 1) % sequence.length;
+    if (!reducedMotion.value) {
+      activeExpressionTimer = window.setTimeout(
+        showNextActiveExpression,
+        ACTIVE_EXPRESSION_DURATION_MS[state]
+      );
+    }
+  }
+
+  function updateReducedMotion(event) {
+    reducedMotion.value = Boolean(event.matches);
+    clearBlinkTimers();
+    clearActiveExpression();
+    clearIdleGesture();
+    blinking.value = false;
+    if (ACTIVE_EXPRESSION_SEQUENCES[expression.value]) {
+      showNextActiveExpression();
+    }
+    if (!reducedMotion.value) {
+      scheduleBlink();
+      scheduleIdleGesture();
+    }
+  }
+
+  watch(expression, () => {
+    clearActiveExpression();
+    clearIdleGesture();
+    if (ACTIVE_EXPRESSION_SEQUENCES[expression.value]) {
+      showNextActiveExpression();
+    } else if (!reducedMotion.value) {
+      scheduleIdleGesture(900 + (Math.random() * 1_200));
+    }
+  });
+
+  onMounted(() => {
+    reducedMotionQuery = window.matchMedia("(prefers-reduced-motion: reduce)");
+    reducedMotion.value = reducedMotionQuery.matches;
+    reducedMotionQuery.addEventListener?.("change", updateReducedMotion);
+    if (!reducedMotion.value) {
+      scheduleBlink(1_800 + (Math.random() * 1_800));
+      scheduleIdleGesture(900 + (Math.random() * 1_500));
+    }
+    if (ACTIVE_EXPRESSION_SEQUENCES[expression.value]) {
+      showNextActiveExpression();
+    }
+  });
+
+  onBeforeUnmount(() => {
+    clearBlinkTimers();
+    clearActiveExpression();
+    clearIdleGesture();
+    reducedMotionQuery?.removeEventListener?.("change", updateReducedMotion);
+    reducedMotionQuery = null;
+  });
+  return { blinking, activeExpression, idleGesture, expression, visibleMouthPose, reducedMotion };
+}

package/src/client/voiceController.js

@@ -0,0 +1,124 @@
+import { effectScope, shallowReactive, watch } from "vue";
+import { useVoiceConversation } from "./voiceConversation.js";
+
+/** One controller per application shell. Bindings retain their own conversation. */
+export function createVoiceConversationController({ connectSpeech, createSession = useVoiceConversation } = {}) {
+  if (typeof connectSpeech !== "function") throw new TypeError("Voice requires an application speech connection.");
+  const state = shallowReactive({ binding: null, session: null, visible: false, nextTarget: null, busy: false, error: "" });
+  let scope;
+  let revision = 0;
+  let operations = Promise.resolve();
+  let disposed = false;
+
+  function serialize(operation) {
+    const result = operations.then(operation);
+    operations = result.catch(() => {});
+    return result;
+  }
+
+  async function release() {
+    const session = state.session;
+    // Keep the old owner visible until microphone and playback cleanup finishes.
+    if (session) await session.close();
+    scope?.stop();
+    scope = null;
+    await state.binding?.release?.();
+    state.binding = null;
+    state.session = null;
+  }
+
+  async function activate(binding, requestRevision) {
+    const socketUrl = await connectSpeech(binding);
+    if (disposed || requestRevision !== revision) return false;
+    if (!socketUrl) throw new Error("A speech service is not configured for this conversation.");
+    await release();
+    if (disposed || requestRevision !== revision) return false;
+    const nextScope = effectScope(true);
+    try {
+      binding.retain?.();
+      const session = nextScope.run(() => createSession(binding, { socketUrl }));
+      scope = nextScope;
+      state.binding = binding;
+      state.session = session;
+      state.visible = true;
+      state.nextTarget = null;
+      nextScope.run(() => watch(() => binding.available, (available) => {
+        if (available !== false) return;
+        state.error = "This conversation is no longer available for voice.";
+        void end({ discard: true });
+      }, { immediate: true }));
+      return true;
+    } catch (error) {
+      nextScope.stop();
+      binding.release?.();
+      throw error;
+    }
+  }
+
+  function open({ conversation } = {}) {
+    if (!conversation?.id || !conversation.state || typeof conversation.submitText !== "function") {
+      return Promise.reject(new TypeError("Voice requires a conversation identity, state and text submission."));
+    }
+    if (disposed) return Promise.reject(new Error("The voice host has been disposed."));
+    const requestRevision = ++revision;
+    return serialize(async () => {
+      if (requestRevision !== revision || disposed) return false;
+      state.error = "";
+      if (state.binding?.id === conversation.id) {
+        state.visible = true;
+        state.nextTarget = null;
+        return true;
+      }
+      if (conversation.available === false) throw new Error("This conversation is not available for voice.");
+      if (state.session?.hasUnsentSpeech.value) {
+        state.nextTarget = conversation;
+        state.visible = true;
+        return false;
+      }
+      state.busy = true;
+      try { return await activate(conversation, requestRevision); }
+      catch (error) { state.error = error.message; throw error; }
+      finally { state.busy = false; }
+    });
+  }
+
+  function switchTarget({ discard = false } = {}) {
+    const target = state.nextTarget;
+    if (!target) return Promise.resolve(false);
+    if (!discard && state.session?.hasUnsentSpeech.value) return Promise.resolve(false);
+    if (state.session?.sending.value) return Promise.resolve(false);
+    const requestRevision = ++revision;
+    return serialize(async () => {
+      state.busy = true;
+      try { return await activate(target, requestRevision); }
+      catch (error) { state.error = error.message; throw error; }
+      finally { state.busy = false; }
+    });
+  }
+
+  function end({ discard = false } = {}) {
+    if (!discard && state.session?.hasUnsentSpeech.value) {
+      state.visible = true;
+      state.error = "Finish or discard your recording before ending voice.";
+      return Promise.resolve(false);
+    }
+    ++revision;
+    return serialize(async () => {
+      state.busy = true;
+      try {
+        await release();
+        state.visible = false;
+        state.nextTarget = null;
+        return true;
+      } finally { state.busy = false; }
+    });
+  }
+
+  return {
+    state, open, switchTarget, end,
+    minimize() { state.visible = false; },
+    reveal() { if (state.session) state.visible = true; },
+    cancelSwitch() { state.nextTarget = null; },
+    async dispose() { disposed = true; await end({ discard: true }); }
+  };
+}