@absolutejs/voice 0.0.22-beta.93 → 0.0.22-beta.94

package/dist/client/htmxBootstrap.js

@@ -1110,6 +1110,352 @@ var createVoiceController = (path, options = {}) => {
   };
 };
 
+// src/client/audioPlayer.ts
+var DEFAULT_LOOKAHEAD_MS = 15;
+var createInitialState3 = () => ({
+  activeSourceCount: 0,
+  error: null,
+  isActive: false,
+  isPlaying: false,
+  lastInterruptLatencyMs: undefined,
+  lastPlaybackStopLatencyMs: undefined,
+  processedChunkCount: 0,
+  queuedChunkCount: 0
+});
+var getAudioContextCtor = () => {
+  if (typeof window === "undefined") {
+    return typeof AudioContext === "undefined" ? undefined : AudioContext;
+  }
+  return window.AudioContext ?? window.webkitAudioContext;
+};
+var decodePCM16LEChunk = (audioContext, chunk) => {
+  const format = chunk.format;
+  if (format.container !== "raw" || format.encoding !== "pcm_s16le") {
+    throw new Error(`Unsupported assistant audio format: ${format.container}/${format.encoding}`);
+  }
+  const bytes = chunk.chunk;
+  const channels = Math.max(1, format.channels);
+  const sampleCount = Math.floor(bytes.byteLength / 2);
+  const frameCount = Math.max(1, Math.floor(sampleCount / channels));
+  const audioBuffer = audioContext.createBuffer(channels, frameCount, format.sampleRateHz);
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  for (let channelIndex = 0;channelIndex < channels; channelIndex += 1) {
+    const channelData = audioBuffer.getChannelData(channelIndex);
+    for (let frameIndex = 0;frameIndex < frameCount; frameIndex += 1) {
+      const sampleIndex = frameIndex * channels + channelIndex;
+      const sampleOffset = sampleIndex * 2;
+      if (sampleOffset + 1 >= bytes.byteLength) {
+        channelData[frameIndex] = 0;
+        continue;
+      }
+      channelData[frameIndex] = view.getInt16(sampleOffset, true) / 32768;
+    }
+  }
+  return audioBuffer;
+};
+var createVoiceAudioPlayer = (source, options = {}) => {
+  const subscribers = new Set;
+  const sourceNodes = new Set;
+  const lookaheadSeconds = (options.lookaheadMs ?? DEFAULT_LOOKAHEAD_MS) / 1000;
+  let state = createInitialState3();
+  let audioContext = null;
+  let outputNode = null;
+  let queueEndTime = 0;
+  let syncPromise = Promise.resolve();
+  let interruptStartedAt = null;
+  let interruptPromise = null;
+  let resolveInterruptPromise = null;
+  let interruptFallbackTimer = null;
+  const notify = () => {
+    for (const subscriber of subscribers) {
+      subscriber();
+    }
+  };
+  const setState = (next) => {
+    state = {
+      ...state,
+      ...next
+    };
+    notify();
+  };
+  const clearError = () => {
+    if (state.error !== null) {
+      setState({ error: null });
+    }
+  };
+  const clearInterruptTimer = () => {
+    if (interruptFallbackTimer !== null) {
+      clearTimeout(interruptFallbackTimer);
+      interruptFallbackTimer = null;
+    }
+  };
+  const resolveInterrupt = (latencyMs) => {
+    clearInterruptTimer();
+    interruptStartedAt = null;
+    setState({
+      activeSourceCount: sourceNodes.size,
+      isPlaying: false,
+      lastInterruptLatencyMs: latencyMs,
+      lastPlaybackStopLatencyMs: state.lastPlaybackStopLatencyMs ?? latencyMs
+    });
+    resolveInterruptPromise?.();
+    resolveInterruptPromise = null;
+    interruptPromise = null;
+  };
+  const estimateOutputStopLatencyMs = (context) => {
+    if (!context) {
+      return 0;
+    }
+    return Math.max(0, ((context.baseLatency ?? 0) + (context.outputLatency ?? 0)) * 1000);
+  };
+  const restoreOutputGain = (context) => {
+    if (!outputNode) {
+      return;
+    }
+    const gainValue = 1;
+    if (outputNode.gain.setValueAtTime) {
+      outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
+      return;
+    }
+    outputNode.gain.value = gainValue;
+  };
+  const muteOutputGain = (context) => {
+    if (!outputNode) {
+      return;
+    }
+    const gainValue = 0;
+    if (outputNode.gain.setValueAtTime) {
+      outputNode.gain.setValueAtTime(gainValue, context?.currentTime ?? 0);
+      return;
+    }
+    outputNode.gain.value = gainValue;
+  };
+  const maybeResolveInterrupt = () => {
+    if (interruptStartedAt === null || sourceNodes.size > 0) {
+      return;
+    }
+    resolveInterrupt(Date.now() - interruptStartedAt);
+  };
+  const ensureAudioContext = async () => {
+    if (audioContext) {
+      return audioContext;
+    }
+    if (options.createAudioContext) {
+      audioContext = options.createAudioContext();
+    } else {
+      const AudioContextCtor = getAudioContextCtor();
+      if (!AudioContextCtor) {
+        throw new Error("Assistant audio playback requires AudioContext support.");
+      }
+      audioContext = new AudioContextCtor;
+    }
+    if (audioContext.createGain) {
+      outputNode = audioContext.createGain();
+      outputNode.connect?.(audioContext.destination);
+    }
+    queueEndTime = audioContext.currentTime;
+    return audioContext;
+  };
+  const scheduleChunk = async (chunk) => {
+    const context = await ensureAudioContext();
+    const buffer = decodePCM16LEChunk(context, chunk);
+    const node = context.createBufferSource();
+    node.buffer = buffer;
+    node.connect(outputNode ?? context.destination);
+    node.onended = () => {
+      sourceNodes.delete(node);
+      node.disconnect?.();
+      setState({
+        activeSourceCount: sourceNodes.size,
+        isPlaying: sourceNodes.size > 0 && state.isActive
+      });
+      maybeResolveInterrupt();
+    };
+    const startAt = Math.max(context.currentTime + lookaheadSeconds, queueEndTime);
+    queueEndTime = startAt + buffer.duration;
+    sourceNodes.add(node);
+    setState({
+      activeSourceCount: sourceNodes.size,
+      isPlaying: true
+    });
+    node.start(startAt);
+  };
+  const stopQueuedPlayback = (options2) => {
+    for (const node of [...sourceNodes]) {
+      node.stop?.();
+    }
+    queueEndTime = audioContext ? audioContext.currentTime : 0;
+    if (options2?.forceClear) {
+      for (const node of sourceNodes) {
+        node.disconnect?.();
+      }
+      sourceNodes.clear();
+      maybeResolveInterrupt();
+    }
+  };
+  const sync = async () => {
+    if (!state.isActive) {
+      return;
+    }
+    const nextChunks = source.assistantAudio.slice(state.processedChunkCount);
+    if (nextChunks.length === 0) {
+      return;
+    }
+    try {
+      clearError();
+      for (const chunk of nextChunks) {
+        await scheduleChunk(chunk);
+      }
+      setState({
+        processedChunkCount: source.assistantAudio.length,
+        queuedChunkCount: state.queuedChunkCount + nextChunks.length
+      });
+    } catch (error) {
+      setState({
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  };
+  const queueSync = () => {
+    syncPromise = syncPromise.then(() => sync(), () => sync());
+    return syncPromise;
+  };
+  const unsubscribeSource = source.subscribe(() => {
+    if (options.autoStart && !state.isActive && source.assistantAudio.length > 0) {
+      player.start();
+      return;
+    }
+    if (state.isActive) {
+      queueSync();
+    }
+  });
+  const player = {
+    close: async () => {
+      unsubscribeSource();
+      stopQueuedPlayback({ forceClear: true });
+      clearInterruptTimer();
+      resolveInterruptPromise?.();
+      resolveInterruptPromise = null;
+      interruptPromise = null;
+      interruptStartedAt = null;
+      if (audioContext && audioContext.state !== "closed") {
+        await audioContext.close();
+      }
+      audioContext = null;
+      outputNode?.disconnect?.();
+      outputNode = null;
+      queueEndTime = 0;
+      setState({
+        activeSourceCount: 0,
+        isActive: false,
+        isPlaying: false
+      });
+    },
+    get activeSourceCount() {
+      return state.activeSourceCount;
+    },
+    get error() {
+      return state.error;
+    },
+    getSnapshot: () => state,
+    get isActive() {
+      return state.isActive;
+    },
+    get isPlaying() {
+      return state.isPlaying;
+    },
+    interrupt: async () => {
+      const startedAt = Date.now();
+      const context = await ensureAudioContext();
+      interruptStartedAt = startedAt;
+      muteOutputGain(context);
+      const playbackStopLatencyMs = Date.now() - startedAt + estimateOutputStopLatencyMs(context);
+      setState({
+        isActive: false,
+        isPlaying: sourceNodes.size > 0,
+        lastPlaybackStopLatencyMs: playbackStopLatencyMs
+      });
+      if (sourceNodes.size === 0) {
+        resolveInterrupt(playbackStopLatencyMs);
+        return;
+      }
+      if (!interruptPromise) {
+        interruptPromise = new Promise((resolve) => {
+          resolveInterruptPromise = resolve;
+        });
+      }
+      clearInterruptTimer();
+      interruptFallbackTimer = setTimeout(() => {
+        for (const node of sourceNodes) {
+          node.disconnect?.();
+        }
+        sourceNodes.clear();
+        resolveInterrupt(Date.now() - startedAt);
+      }, 250);
+      stopQueuedPlayback();
+      await interruptPromise;
+    },
+    get lastInterruptLatencyMs() {
+      return state.lastInterruptLatencyMs;
+    },
+    get lastPlaybackStopLatencyMs() {
+      return state.lastPlaybackStopLatencyMs;
+    },
+    pause: async () => {
+      if (!audioContext) {
+        setState({
+          activeSourceCount: 0,
+          isActive: false,
+          isPlaying: false
+        });
+        return;
+      }
+      await audioContext.suspend();
+      setState({
+        activeSourceCount: sourceNodes.size,
+        isActive: false,
+        isPlaying: false
+      });
+    },
+    get processedChunkCount() {
+      return state.processedChunkCount;
+    },
+    get queuedChunkCount() {
+      return state.queuedChunkCount;
+    },
+    start: async () => {
+      try {
+        clearError();
+        const context = await ensureAudioContext();
+        restoreOutputGain(context);
+        if (context.state === "suspended") {
+          await context.resume();
+        }
+        setState({
+          activeSourceCount: sourceNodes.size,
+          isActive: true,
+          isPlaying: context.state === "running"
+        });
+        await queueSync();
+      } catch (error) {
+        setState({
+          error: error instanceof Error ? error.message : String(error),
+          isActive: false,
+          isPlaying: false
+        });
+        throw error;
+      }
+    },
+    subscribe: (subscriber) => {
+      subscribers.add(subscriber);
+      return () => {
+        subscribers.delete(subscriber);
+      };
+    }
+  };
+  return player;
+};
+
 // src/client/bargeInMonitor.ts
 var DEFAULT_THRESHOLD_MS = 250;
 var createEventId = () => `barge-in:${Date.now()}:${crypto.randomUUID?.() ?? Math.random().toString(36).slice(2)}`;
@@ -1181,6 +1527,58 @@ var createVoiceBargeInMonitor = (options = {}) => {
   };
 };
 
+// src/client/duplex.ts
+var DEFAULT_INTERRUPT_THRESHOLD = 0.08;
+var shouldInterruptForLevel = (level, options = {}) => (options.enabled ?? true) && level >= (options.interruptThreshold ?? DEFAULT_INTERRUPT_THRESHOLD);
+var bindVoiceBargeIn = (controller, player, options = {}) => {
+  let lastPartial = controller.partial;
+  const interruptIfPlaying = (reason) => {
+    if (!player.isPlaying || options.enabled === false) {
+      options.monitor?.recordSkipped({
+        reason,
+        sessionId: controller.sessionId
+      });
+      return;
+    }
+    options.monitor?.recordRequested({
+      reason,
+      sessionId: controller.sessionId
+    });
+    player.interrupt().then(() => {
+      options.monitor?.recordStopped({
+        latencyMs: player.lastInterruptLatencyMs,
+        playbackStopLatencyMs: player.lastPlaybackStopLatencyMs,
+        reason,
+        sessionId: controller.sessionId
+      });
+    });
+  };
+  const unsubscribe = controller.subscribe(() => {
+    if (options.interruptOnPartial === false) {
+      lastPartial = controller.partial;
+      return;
+    }
+    if (!lastPartial && controller.partial) {
+      interruptIfPlaying("partial-transcript");
+    }
+    lastPartial = controller.partial;
+  });
+  return {
+    close: () => {
+      unsubscribe();
+    },
+    handleLevel: (level) => {
+      if (shouldInterruptForLevel(level, options)) {
+        interruptIfPlaying("input-level");
+      }
+    },
+    sendAudio: (audio) => {
+      interruptIfPlaying("manual-audio");
+      controller.sendAudio(audio);
+    }
+  };
+};
+
 // src/client/htmxBootstrap.ts
 var VOICE_WAVE_POINTS = 48;
 var VOICE_WAVE_WIDTH = 320;
@@ -1378,44 +1776,19 @@ var initVoiceHTMXRoot = (root) => {
   let isCapturing = false;
   let micError = null;
   let waveLevels = createInitialVoiceWaveLevels();
-  let lastInputLevel = 0;
-  let lastAssistantAt = 0;
-  let lastAssistantAudioCount = 0;
-  let lastAssistantTextCount = 0;
-  const syncBargeInOutput = () => {
-    if (!bargeInMonitor) {
-      return;
-    }
-    const voice = currentVoice();
-    const audioCount = voice.assistantAudio.length;
-    const textCount = voice.assistantTexts.length;
-    if (audioCount > lastAssistantAudioCount || textCount > lastAssistantTextCount) {
-      lastAssistantAt = Date.now();
-    }
-    lastAssistantAudioCount = audioCount;
-    lastAssistantTextCount = textCount;
-  };
-  const sendAudioWithBargeInEvidence = (audio, sendAudio) => {
-    syncBargeInOutput();
-    if (bargeInMonitor && Date.now() - lastAssistantAt <= bargeInRecentWindowMs && lastInputLevel >= bargeInSpeechThreshold) {
-      bargeInMonitor.recordRequested({
-        reason: "manual-audio",
-        sessionId: currentVoice().sessionId
-      });
-      bargeInMonitor.recordStopped({
-        latencyMs: 0,
-        playbackStopLatencyMs: 0,
-        reason: "manual-audio",
-        sessionId: currentVoice().sessionId
-      });
-    }
-    sendAudio(audio);
-  };
+  let guidedBargeInBinding = null;
+  let generalBargeInBinding = null;
   const guidedVoice = createVoiceController(guidedPath, {
     capture: {
-      onAudio: sendAudioWithBargeInEvidence,
+      onAudio: (audio, sendAudio) => {
+        if (guidedBargeInBinding) {
+          guidedBargeInBinding.sendAudio(audio);
+          return;
+        }
+        sendAudio(audio);
+      },
       onLevel: (level) => {
-        lastInputLevel = level;
+        guidedBargeInBinding?.handleLevel(level);
         waveLevels = pushVoiceWaveLevel(waveLevels, level);
         renderWave();
       }
@@ -1424,9 +1797,15 @@ var initVoiceHTMXRoot = (root) => {
   });
   const generalVoice = createVoiceController(generalPath, {
     capture: {
-      onAudio: sendAudioWithBargeInEvidence,
+      onAudio: (audio, sendAudio) => {
+        if (generalBargeInBinding) {
+          generalBargeInBinding.sendAudio(audio);
+          return;
+        }
+        sendAudio(audio);
+      },
       onLevel: (level) => {
-        lastInputLevel = level;
+        generalBargeInBinding?.handleLevel(level);
         waveLevels = pushVoiceWaveLevel(waveLevels, level);
         renderWave();
       }
@@ -1435,7 +1814,18 @@ var initVoiceHTMXRoot = (root) => {
   });
   const stopGuidedBinding = guidedVoice.bindHTMX({ element: syncElement });
   const stopGeneralBinding = generalVoice.bindHTMX({ element: syncElement });
+  const guidedAudioPlayer = createVoiceAudioPlayer(guidedVoice);
+  const generalAudioPlayer = createVoiceAudioPlayer(generalVoice);
+  guidedBargeInBinding = bindVoiceBargeIn(guidedVoice, guidedAudioPlayer, {
+    interruptThreshold: bargeInSpeechThreshold,
+    monitor: bargeInMonitor ?? undefined
+  });
+  generalBargeInBinding = bindVoiceBargeIn(generalVoice, generalAudioPlayer, {
+    interruptThreshold: bargeInSpeechThreshold,
+    monitor: bargeInMonitor ?? undefined
+  });
   const currentVoice = () => activeMode === "general" ? generalVoice : guidedVoice;
+  const currentAudioPlayer = () => activeMode === "general" ? generalAudioPlayer : guidedAudioPlayer;
   const renderWave = () => {
     const path = createVoiceWavePath(waveLevels);
     voiceWaveGlow.setAttribute("d", path);
@@ -1514,11 +1904,15 @@ var initVoiceHTMXRoot = (root) => {
     }
   };
   guidedVoice.subscribe(() => {
-    syncBargeInOutput();
+    if (guidedVoice.assistantAudio.length > 0) {
+      guidedAudioPlayer.start().catch(() => {});
+    }
     render();
   });
   generalVoice.subscribe(() => {
-    syncBargeInOutput();
+    if (generalVoice.assistantAudio.length > 0) {
+      generalAudioPlayer.start().catch(() => {});
+    }
     render();
   });
   startGuidedButton.addEventListener("click", () => {
@@ -1533,6 +1927,10 @@ var initVoiceHTMXRoot = (root) => {
   window.addEventListener("beforeunload", () => {
     guidedVoice.stopRecording();
     generalVoice.stopRecording();
+    guidedBargeInBinding?.close();
+    generalBargeInBinding?.close();
+    guidedAudioPlayer.close();
+    generalAudioPlayer.close();
     stopGuidedBinding();
     stopGeneralBinding();
     guidedVoice.close();

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.93",
+	"version": "0.0.22-beta.94",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",