@nextpa/chat-core 0.1.0

package/dist/services/realtime/google/google-audio.js

@@ -0,0 +1,410 @@
+import { logger } from '../../../utils/logger';
+const CAPTURE_SAMPLE_RATE = 16000;
+const PLAYBACK_SAMPLE_RATE = 24000;
+const PLAYBACK_BUFFER_SIZE = 7680;
+const INITIAL_BUFFER_TIME = 0.1;
+const SCHEDULE_AHEAD_TIME = 0.2;
+const FADE_OUT_DURATION = 0.1;
+const FADE_OUT_RESET_DELAY_MS = 200;
+const SPEECH_RMS_THRESHOLD = 1500;
+const SPEECH_START_DEBOUNCE_MS = 150;
+const SPEECH_END_DEBOUNCE_MS = 1500;
+export function resolveWorkletUrl() {
+    const loaderBase = globalThis.__nextpaWidgetBaseUrl;
+    if (loaderBase) {
+        return loaderBase + 'audio-capture-worklet.js';
+    }
+    // Resolve against the page origin instead of import.meta.url. In the
+    // embedded widget the chunk loads from a CDN-style absolute URL and the
+    // module-URL fallback works; in the chat-app the Angular esbuild dev
+    // server can hand back module URLs the URL constructor refuses to use
+    // as a base, which surfaces as `AbortError: Unable to load a worklet's
+    // module.` at AudioWorklet.addModule(). The worklet is shipped from the
+    // same origin as the app shell, so window.location.origin is correct
+    // for both dev and prod builds.
+    if (globalThis.window?.location?.origin) {
+        return `${globalThis.window.location.origin}/audio-capture-worklet.js`;
+    }
+    return new URL('/audio-capture-worklet.js', import.meta.url).href;
+}
+export class GoogleAudioHandler {
+    captureContext = null;
+    mediaStream = null;
+    workletNode = null;
+    playbackContext = null;
+    gainNode = null;
+    aecStreamDestination = null;
+    loopbackSource = null;
+    loopbackSink = null;
+    playbackElement = null;
+    scheduledTime = 0;
+    audioQueue = [];
+    isMuted = false;
+    isPlaying = false;
+    capturePaused = false;
+    speechDetected = false;
+    silenceTimerId = null;
+    speechStartTimerId = null;
+    consecutiveSpeechFrames = 0;
+    onAudioData = null;
+    onSpeechStarted = null;
+    onSpeechEnded = null;
+    onAudioLevel = null;
+    get isPlaybackActive() {
+        if (this.audioQueue.length > 0 || this.isPlaying)
+            return true;
+        if (!this.playbackContext)
+            return false;
+        return this.scheduledTime > this.playbackContext.currentTime;
+    }
+    async startCapture() {
+        this.mediaStream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                sampleRate: CAPTURE_SAMPLE_RATE,
+                channelCount: 1,
+                echoCancellation: true,
+                noiseSuppression: true,
+                autoGainControl: true,
+            },
+        });
+        this.captureContext = new AudioContext({ sampleRate: CAPTURE_SAMPLE_RATE });
+        // Browsers (Chrome especially) hand back AudioContexts in the
+        // "suspended" state until an explicit resume from a user-gesture
+        // promise chain. Without this the worklet receives no audio frames
+        // and the remote side hears silence.
+        const stateBeforeResume = this.captureContext.state;
+        if (this.captureContext.state === 'suspended') {
+            await this.captureContext.resume();
+        }
+        logger.debug('[GoogleAudio] Capture context state:', stateBeforeResume, '→', this.captureContext.state, 'sampleRate:', this.captureContext.sampleRate);
+        const source = this.captureContext.createMediaStreamSource(this.mediaStream);
+        await this.captureContext.audioWorklet.addModule(resolveWorkletUrl());
+        this.workletNode = new AudioWorkletNode(this.captureContext, 'audio-capture-processor');
+        let frameCount = 0;
+        this.workletNode.port.onmessage = (event) => {
+            const int16Buffer = new Int16Array(event.data.buffer);
+            this.detectSpeech(int16Buffer);
+            // First frame and every 100th frame: log a heartbeat so we can
+            // see in the console whether mic data is actually flowing and why
+            // it would be dropped (muted / paused / no callback wired up).
+            frameCount++;
+            if (frameCount === 1 || frameCount % 100 === 0) {
+                logger.debug('[GoogleAudio] Mic frame', frameCount, 'len:', int16Buffer.length, 'isMuted:', this.isMuted, 'capturePaused:', this.capturePaused, 'onAudioData:', this.onAudioData ? 'wired' : 'NULL');
+            }
+            if (this.isMuted || this.capturePaused || !this.onAudioData) {
+                return;
+            }
+            this.onAudioData(arrayBufferToBase64(int16Buffer.buffer));
+        };
+        source.connect(this.workletNode);
+        this.workletNode.connect(this.captureContext.destination);
+        await this.setupPlaybackChain();
+    }
+    playAudio(base64Pcm) {
+        if (!this.playbackContext || !this.gainNode)
+            return;
+        if (this.playbackContext.state === 'suspended') {
+            void this.playbackContext.resume();
+        }
+        void this.playbackElement?.play().catch(() => undefined);
+        const pcmBytes = base64ToUint8Array(base64Pcm);
+        const float32 = pcm16ToFloat32(pcmBytes);
+        this.enqueueAudio(float32);
+    }
+    async setupPlaybackChain() {
+        if (this.playbackContext)
+            return;
+        this.playbackContext = new AudioContext({ sampleRate: PLAYBACK_SAMPLE_RATE });
+        // Same suspended-context trap as the capture side: created after the
+        // user-gesture chain yielded, the context queues the greeting audio
+        // without ever playing it — the assistant appears mute until a reset.
+        const playbackStateBefore = this.playbackContext.state;
+        if (this.playbackContext.state === 'suspended') {
+            await this.playbackContext.resume();
+        }
+        logger.debug('[GoogleAudio] Playback context state:', playbackStateBefore, '→', this.playbackContext.state);
+        this.gainNode = this.playbackContext.createGain();
+        this.scheduledTime = 0;
+        // Route playback through a local WebRTC loopback into a hidden <audio>
+        // element instead of straight to AudioContext.destination. The mic is
+        // captured with echoCancellation:true, but the browser's AEC only
+        // subtracts audio it sees on its WebRTC render path — Web-Audio output
+        // to destination is invisible to it. On speakerphone the mic then picks
+        // up the assistant's own voice and Gemini's server VAD interrupts the
+        // model mid-sentence. Feeding playback through a PeerConnection makes it
+        // a far-end reference the AEC can cancel. Falls back to direct output
+        // when WebRTC is unavailable.
+        await this.tryRouteThroughAecLoopback();
+        this.connectGainOutput();
+    }
+    connectGainOutput() {
+        if (!this.playbackContext || !this.gainNode)
+            return;
+        if (this.aecStreamDestination) {
+            this.gainNode.connect(this.aecStreamDestination);
+        }
+        else {
+            this.gainNode.connect(this.playbackContext.destination);
+        }
+    }
+    async tryRouteThroughAecLoopback() {
+        if (typeof RTCPeerConnection === 'undefined' || typeof document === 'undefined'
+            || !this.playbackContext) {
+            return;
+        }
+        try {
+            const streamDestination = this.playbackContext.createMediaStreamDestination();
+            const source = new RTCPeerConnection();
+            const sink = new RTCPeerConnection();
+            source.addEventListener('icecandidate', (e) => {
+                if (e.candidate)
+                    void sink.addIceCandidate(e.candidate);
+            });
+            sink.addEventListener('icecandidate', (e) => {
+                if (e.candidate)
+                    void source.addIceCandidate(e.candidate);
+            });
+            const loopbackStream = new MediaStream();
+            sink.addEventListener('track', (e) => loopbackStream.addTrack(e.track));
+            for (const track of streamDestination.stream.getTracks()) {
+                source.addTrack(track, streamDestination.stream);
+            }
+            const offer = await source.createOffer();
+            await source.setLocalDescription(offer);
+            await sink.setRemoteDescription(offer);
+            const answer = await sink.createAnswer();
+            await sink.setLocalDescription(answer);
+            await source.setRemoteDescription(answer);
+            const element = document.createElement('audio');
+            element.autoplay = true;
+            element.setAttribute('playsinline', '');
+            element.style.display = 'none';
+            element.srcObject = loopbackStream;
+            document.body.appendChild(element);
+            await element.play().catch(() => undefined);
+            this.aecStreamDestination = streamDestination;
+            this.loopbackSource = source;
+            this.loopbackSink = sink;
+            this.playbackElement = element;
+            logger.debug('[GoogleAudio] Playback routed through AEC loopback');
+        }
+        catch (err) {
+            logger.debug('[GoogleAudio] AEC loopback unavailable, using direct output:', err);
+            this.teardownAecLoopback();
+        }
+    }
+    teardownAecLoopback() {
+        this.loopbackSource?.close();
+        this.loopbackSink?.close();
+        this.loopbackSource = null;
+        this.loopbackSink = null;
+        if (this.playbackElement) {
+            this.playbackElement.srcObject = null;
+            this.playbackElement.remove();
+            this.playbackElement = null;
+        }
+        this.aecStreamDestination = null;
+    }
+    enqueueAudio(samples) {
+        for (let offset = 0; offset < samples.length; offset += PLAYBACK_BUFFER_SIZE) {
+            const end = Math.min(offset + PLAYBACK_BUFFER_SIZE, samples.length);
+            this.audioQueue.push(samples.subarray(offset, end));
+        }
+        if (!this.isPlaying) {
+            this.isPlaying = true;
+            this.schedulePlayback();
+        }
+    }
+    schedulePlayback() {
+        if (!this.playbackContext || !this.gainNode) {
+            return;
+        }
+        const currentTime = this.playbackContext.currentTime;
+        if (this.scheduledTime < currentTime) {
+            this.scheduledTime = currentTime + INITIAL_BUFFER_TIME;
+        }
+        while (this.audioQueue.length > 0 && this.scheduledTime < currentTime + SCHEDULE_AHEAD_TIME) {
+            const chunk = this.audioQueue.shift();
+            const buffer = this.playbackContext.createBuffer(1, chunk.length, PLAYBACK_SAMPLE_RATE);
+            buffer.getChannelData(0).set(chunk);
+            const source = this.playbackContext.createBufferSource();
+            source.buffer = buffer;
+            source.connect(this.gainNode);
+            source.start(this.scheduledTime);
+            this.scheduledTime += buffer.duration;
+        }
+        if (this.audioQueue.length > 0) {
+            const remainingTime = Math.max(0, (this.scheduledTime - currentTime - SCHEDULE_AHEAD_TIME) * 1000);
+            setTimeout(() => this.schedulePlayback(), Math.min(remainingTime, 100));
+        }
+        else {
+            this.isPlaying = false;
+        }
+    }
+    resume() {
+        if (this.captureContext?.state === 'suspended') {
+            void this.captureContext.resume().catch(() => undefined);
+        }
+        if (this.playbackContext?.state === 'suspended') {
+            void this.playbackContext.resume().catch(() => undefined);
+        }
+        void this.playbackElement?.play().catch(() => undefined);
+    }
+    pauseCapture() {
+        this.capturePaused = true;
+    }
+    resumeCapture() {
+        this.capturePaused = false;
+    }
+    setMuted(muted) {
+        this.isMuted = muted;
+        if (this.mediaStream) {
+            for (const track of this.mediaStream.getAudioTracks()) {
+                track.enabled = !muted;
+            }
+        }
+    }
+    stop() {
+        this.workletNode?.disconnect();
+        this.workletNode = null;
+        if (this.mediaStream) {
+            for (const track of this.mediaStream.getTracks()) {
+                track.stop();
+            }
+        }
+        this.captureContext?.close();
+        this.playbackContext?.close();
+        this.teardownAecLoopback();
+        this.mediaStream = null;
+        this.captureContext = null;
+        this.playbackContext = null;
+        this.gainNode = null;
+        this.scheduledTime = 0;
+        this.audioQueue = [];
+        this.isPlaying = false;
+        this.capturePaused = false;
+        this.speechDetected = false;
+        this.consecutiveSpeechFrames = 0;
+        if (this.silenceTimerId) {
+            clearTimeout(this.silenceTimerId);
+            this.silenceTimerId = null;
+        }
+        if (this.speechStartTimerId) {
+            clearTimeout(this.speechStartTimerId);
+            this.speechStartTimerId = null;
+        }
+    }
+    detectSpeech(samples) {
+        if (this.isMuted)
+            return;
+        if (!this.onSpeechStarted && !this.onSpeechEnded && !this.onAudioLevel)
+            return;
+        let sumSquares = 0;
+        for (const sample of samples) {
+            sumSquares += sample * sample;
+        }
+        const rms = Math.sqrt(sumSquares / samples.length);
+        // Int16 PCM range — normalise into 0..1 against the codec full scale so
+        // the dBFS mapping downstream in voice-state-manager is meaningful.
+        this.onAudioLevel?.(Math.min(1, rms / 32768));
+        if (rms > SPEECH_RMS_THRESHOLD) {
+            this.handleSpeechFrame();
+        }
+        else {
+            this.handleSilenceFrame();
+        }
+    }
+    handleSpeechFrame() {
+        if (this.silenceTimerId) {
+            clearTimeout(this.silenceTimerId);
+            this.silenceTimerId = null;
+        }
+        if (this.speechDetected)
+            return;
+        this.consecutiveSpeechFrames++;
+        this.speechStartTimerId ??= setTimeout(() => {
+            this.speechStartTimerId = null;
+            if (this.consecutiveSpeechFrames > 0 && !this.speechDetected) {
+                this.speechDetected = true;
+                this.onSpeechStarted?.();
+            }
+        }, SPEECH_START_DEBOUNCE_MS);
+    }
+    handleSilenceFrame() {
+        this.consecutiveSpeechFrames = 0;
+        if (this.speechStartTimerId) {
+            clearTimeout(this.speechStartTimerId);
+            this.speechStartTimerId = null;
+        }
+        if (this.speechDetected && !this.silenceTimerId) {
+            this.silenceTimerId = setTimeout(() => {
+                this.silenceTimerId = null;
+                this.speechDetected = false;
+                this.onSpeechEnded?.();
+            }, SPEECH_END_DEBOUNCE_MS);
+        }
+    }
+    async restartCapture() {
+        const wasMuted = this.isMuted;
+        const callback = this.onAudioData;
+        this.workletNode?.disconnect();
+        this.workletNode = null;
+        if (this.mediaStream) {
+            for (const track of this.mediaStream.getTracks()) {
+                track.stop();
+            }
+        }
+        this.captureContext?.close();
+        this.mediaStream = null;
+        this.captureContext = null;
+        this.onAudioData = callback;
+        await this.startCapture();
+        this.isMuted = wasMuted;
+        const activeStream = this.mediaStream;
+        if (activeStream) {
+            for (const track of activeStream.getAudioTracks()) {
+                track.enabled = !wasMuted;
+            }
+        }
+    }
+    clearPlaybackQueue() {
+        this.audioQueue = [];
+        this.isPlaying = false;
+        if (this.playbackContext && this.gainNode) {
+            const currentTime = this.playbackContext.currentTime;
+            this.gainNode.gain.linearRampToValueAtTime(0, currentTime + FADE_OUT_DURATION);
+            setTimeout(() => {
+                if (this.playbackContext && this.gainNode) {
+                    this.gainNode.disconnect();
+                    this.gainNode = this.playbackContext.createGain();
+                    this.connectGainOutput();
+                    this.scheduledTime = 0;
+                }
+            }, FADE_OUT_RESET_DELAY_MS);
+        }
+    }
+}
+function arrayBufferToBase64(buffer) {
+    const bytes = new Uint8Array(buffer);
+    let binary = '';
+    for (const byte of bytes) {
+        binary += String.fromCodePoint(byte);
+    }
+    return btoa(binary);
+}
+function base64ToUint8Array(base64) {
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.codePointAt(i) ?? 0;
+    }
+    return bytes;
+}
+function pcm16ToFloat32(pcmBytes) {
+    const dataView = new DataView(pcmBytes.buffer);
+    const float32 = new Float32Array(pcmBytes.length / 2);
+    for (let i = 0; i < float32.length; i++) {
+        float32[i] = dataView.getInt16(i * 2, true) / 32768;
+    }
+    return float32;
+}

package/dist/services/realtime/google/google-provider.d.ts

@@ -0,0 +1,47 @@
+import type { RealtimeProviderClient, RealtimeProviderConnectConfig, ProviderEventMap } from '../types';
+export declare class GoogleRealtimeProvider implements RealtimeProviderClient {
+    private session;
+    private readonly audioHandler;
+    private readonly eventHandlers;
+    private _status;
+    private readonly pendingFunctionCalls;
+    private awaitingToolResponse;
+    private isSetupComplete;
+    private modelResponding;
+    private userSpeaking;
+    private pendingCommands;
+    private outputTranscriptBuffer;
+    private inputTranscriptBuffer;
+    private lastCompletedToolCalls;
+    constructor();
+    get status(): 'disconnected' | 'connecting' | 'connected';
+    connect(config: RealtimeProviderConnectConfig): Promise<void>;
+    close(): void;
+    resumeAudio(): void;
+    mute(muted: boolean): void;
+    interrupt(): void;
+    requestResponse(): void;
+    sendToolResult(callId: string, name: string, output: string): void;
+    sendText(text: string): void;
+    waitForAudioPlaybackEnd(timeoutMs?: number): Promise<string>;
+    refreshAudioDevice(): Promise<void>;
+    on<K extends keyof ProviderEventMap>(event: K, handler: ProviderEventMap[K]): () => void;
+    off<K extends keyof ProviderEventMap>(event: K, handler: ProviderEventMap[K]): void;
+    private emit;
+    private resetSessionState;
+    private executeRequestResponse;
+    private executeSendToolResult;
+    private finalizeInputTranscript;
+    private handleServerMessage;
+    private extractAndEmitUsage;
+    private flushPendingCommands;
+    private handleVoiceActivity;
+    private handleServerContent;
+    private static readonly THINKING_RENDER_DELAY_MS;
+    private deferAssistantTranscript;
+    private ensureSpeechStopped;
+    private handleModelTurn;
+    private handleTurnComplete;
+    private handleInterrupted;
+    private handleToolCall;
+}