@nextpa/chat-core 0.1.0

package/dist/services/realtime/openai/openai-provider.js

@@ -0,0 +1,579 @@
+import { OpenAIRealtimeWebRTC } from '@openai/agents-realtime';
+import { mapToolsForOpenAI } from './openai-tool-mapper';
+import { waitForAudioPlaybackEnd } from './openai-audio-monitor';
+import { logger } from '../../../utils/logger';
+function unwrapTranscript(text) {
+    try {
+        const parsed = JSON.parse(text);
+        if (typeof parsed === 'string')
+            return parsed;
+        if (typeof parsed !== 'object' || parsed === null)
+            return text;
+        const transcript = parsed.transcript;
+        if (typeof transcript === 'string')
+            return transcript;
+    }
+    catch {
+        // not JSON — treat as plain text
+    }
+    return text;
+}
+function createPlaybackElement() {
+    if (typeof document === 'undefined')
+        return null;
+    const el = document.createElement('audio');
+    el.autoplay = true;
+    // playsinline keeps iOS from hijacking WebRTC audio into a fullscreen player.
+    el.setAttribute('playsinline', '');
+    el.style.display = 'none';
+    document.body.appendChild(el);
+    return el;
+}
+const TRANSCRIPTION_PURPOSE = 'user_transcription';
+const DEFAULT_TRANSCRIBE_MODEL = 'gpt-4o-transcribe';
+const OUTPUT_AUDIO_PLAYBACK_TIMEOUT_MS = 20000;
+export class OpenAiRealtimeProvider {
+    client;
+    eventHandlers = new Map();
+    oobResponseIds = new Set();
+    oobResponseBuffers = new Map();
+    transcriptionConfig;
+    // WebRTC delivers `output_audio_buffer.started/stopped` over the data channel.
+    // These track *playback* (when the user actually hears audio), unlike
+    // `response.output_audio.done` which only signals generation finished — so
+    // they are the reliable "assistant finished speaking" signal.
+    outputAudioPlaying = false;
+    sawOutputAudioBufferEvents = false;
+    playbackEndWaiters = new Set();
+    analyserContext = null;
+    analyserNode = null;
+    analyserRafId = null;
+    // We own the playback element (instead of letting the SDK create a detached
+    // one) so it can be re-played after iOS pauses it on PWA backgrounding.
+    audioElement;
+    constructor() {
+        this.audioElement = createPlaybackElement();
+        this.client = new OpenAIRealtimeWebRTC(this.audioElement ? { audioElement: this.audioElement } : {});
+    }
+    get status() {
+        return this.client.status;
+    }
+    async connect(config) {
+        this.transcriptionConfig = config.transcription;
+        this.setupInternalEventHandlers();
+        const audioInput = {
+            noiseReduction: { type: 'near_field' },
+            turnDetection: {
+                type: 'semantic_vad',
+                eagerness: 'low',
+                createResponse: true,
+                interruptResponse: true,
+            },
+        };
+        if (config.transcription?.mode === 'native') {
+            audioInput.transcription = {
+                model: config.transcription.model || DEFAULT_TRANSCRIBE_MODEL,
+                language: config.transcription.language,
+                ...(config.transcription.prompt ? { prompt: config.transcription.prompt } : {}),
+            };
+        }
+        await this.client.connect({
+            apiKey: config.apiKey,
+            model: config.model,
+            initialSessionConfig: {
+                instructions: config.instructions,
+                tools: mapToolsForOpenAI(config.tools),
+                toolChoice: config.toolChoice,
+                audio: {
+                    input: audioInput,
+                    output: {
+                        voice: config.voice || 'ash',
+                        speed: config.speed ?? 1,
+                    },
+                },
+            },
+        });
+    }
+    close() {
+        this.stopAudioLevelAnalysis();
+        this.client.close();
+        this.audioElement?.remove();
+        this.oobResponseIds.clear();
+        this.oobResponseBuffers.clear();
+        this.outputAudioPlaying = false;
+        this.resolvePlaybackEndWaiters('closed');
+    }
+    resumeAudio() {
+        if (this.analyserContext?.state === 'suspended') {
+            void this.analyserContext.resume().catch(() => undefined);
+        }
+        if (this.audioElement?.paused) {
+            void this.audioElement.play().catch(() => undefined);
+        }
+    }
+    mute(muted) {
+        this.client.mute(muted);
+    }
+    interrupt() {
+        this.client.interrupt();
+        this.outputAudioPlaying = false;
+        this.resolvePlaybackEndWaiters('interrupted');
+    }
+    requestResponse() {
+        this.client.sendEvent({ type: 'response.create' });
+    }
+    sendToolResult(callId, name, output) {
+        try {
+            this.client.sendEvent({
+                type: 'conversation.item.create',
+                item: { type: 'function_call_output', call_id: callId, output },
+            });
+            this.client.sendEvent({ type: 'response.create' });
+            logger.debug(`[OpenAI] Tool result sent: ${name}`);
+        }
+        catch {
+            logger.debug(`[OpenAI] Failed to send tool result for ${name} (connection may be closing)`);
+        }
+    }
+    sendText(text) {
+        this.client.sendEvent({
+            type: 'conversation.item.create',
+            item: {
+                type: 'message',
+                role: 'user',
+                content: [{ type: 'input_text', text }],
+            },
+        });
+        this.client.sendEvent({ type: 'response.create' });
+    }
+    async waitForAudioPlaybackEnd(timeoutMs = OUTPUT_AUDIO_PLAYBACK_TIMEOUT_MS) {
+        // Preferred: the WebRTC data channel reports exactly when playback ends.
+        if (this.sawOutputAudioBufferEvents) {
+            if (!this.outputAudioPlaying) {
+                return 'output audio idle';
+            }
+            return new Promise((resolve) => {
+                let settled = false;
+                const finish = (reason) => {
+                    if (settled)
+                        return;
+                    settled = true;
+                    this.playbackEndWaiters.delete(finish);
+                    resolve(reason);
+                };
+                this.playbackEndWaiters.add(finish);
+                setTimeout(() => finish('playback wait timeout'), timeoutMs);
+            });
+        }
+        // Fallback for API surfaces without output_audio_buffer.*: RTP silence.
+        const peerConnection = this.getPeerConnection();
+        if (!peerConnection) {
+            return 'no peer connection';
+        }
+        return waitForAudioPlaybackEnd(peerConnection, timeoutMs);
+    }
+    resolvePlaybackEndWaiters(reason) {
+        const waiters = [...this.playbackEndWaiters];
+        this.playbackEndWaiters.clear();
+        waiters.forEach((resolve) => resolve(reason));
+    }
+    async refreshAudioDevice() {
+        const peerConnection = this.getPeerConnection();
+        if (!peerConnection)
+            return;
+        const audioSender = peerConnection.getSenders().find((s) => s.track?.kind === 'audio');
+        if (!audioSender?.track)
+            return;
+        const currentDeviceId = audioSender.track.getSettings().deviceId;
+        try {
+            const newStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            const newTrack = newStream.getAudioTracks()[0];
+            const newDeviceId = newTrack.getSettings().deviceId;
+            if (currentDeviceId === newDeviceId) {
+                newTrack.stop();
+                logger.debug('[OpenAI] Audio device unchanged after devicechange event');
+                return;
+            }
+            const oldTrack = audioSender.track;
+            await audioSender.replaceTrack(newTrack);
+            oldTrack.stop();
+            logger.debug(`[OpenAI] Audio device refreshed: ${currentDeviceId} → ${newDeviceId}`);
+        }
+        catch (err) {
+            logger.debug('[OpenAI] Failed to refresh audio device:', err);
+        }
+    }
+    on(event, handler) {
+        if (!this.eventHandlers.has(event)) {
+            this.eventHandlers.set(event, new Set());
+        }
+        this.eventHandlers.get(event).add(handler);
+        return () => this.off(event, handler);
+    }
+    off(event, handler) {
+        this.eventHandlers.get(event)?.delete(handler);
+    }
+    emit(event, ...args) {
+        const handlers = this.eventHandlers.get(event);
+        if (!handlers)
+            return;
+        for (const handler of handlers) {
+            handler(...args);
+        }
+    }
+    getPeerConnection() {
+        const state = this.client.connectionState;
+        return state.status === 'connected' ? state.peerConnection : null;
+    }
+    // The SDK opens the mic with a bare getUserMedia({ audio: true }), so the
+    // browser's noise/echo/gain processing isn't guaranteed on — on mobile that
+    // let background noise through to the VAD and the assistant kept reacting to
+    // it. Force the processing on for the live mic track once connected.
+    async applyMicNoiseConstraints() {
+        const peerConnection = this.getPeerConnection();
+        const track = peerConnection?.getSenders().find((s) => s.track?.kind === 'audio')?.track;
+        if (!track)
+            return;
+        try {
+            await track.applyConstraints({
+                echoCancellation: true,
+                noiseSuppression: true,
+                autoGainControl: true,
+            });
+            logger.debug('[OpenAI] Mic noise constraints applied');
+        }
+        catch (err) {
+            logger.debug('[OpenAI] Failed to apply mic noise constraints:', err);
+        }
+    }
+    startAudioLevelAnalysis() {
+        const peerConnection = this.getPeerConnection();
+        if (!peerConnection)
+            return;
+        const audioSender = peerConnection.getSenders().find((s) => s.track?.kind === 'audio');
+        if (!audioSender?.track) {
+            logger.debug('[OpenAI] No mic track yet — audio level analysis skipped');
+            return;
+        }
+        try {
+            this.analyserContext = new AudioContext();
+            const stream = new MediaStream([audioSender.track]);
+            const source = this.analyserContext.createMediaStreamSource(stream);
+            this.analyserNode = this.analyserContext.createAnalyser();
+            // 1024 samples ≈ 21 ms at 48 kHz — fast enough to react to syllables
+            // but stable enough for time-domain RMS. The downstream envelope
+            // follower in voice-state-manager owns the actual smoothing, so we
+            // pick a modest internal coefficient.
+            this.analyserNode.fftSize = 1024;
+            this.analyserNode.smoothingTimeConstant = 0.4;
+            source.connect(this.analyserNode);
+            const buffer = new Float32Array(this.analyserNode.fftSize);
+            const tick = () => {
+                if (!this.analyserNode)
+                    return;
+                this.analyserNode.getFloatTimeDomainData(buffer);
+                let sumSquares = 0;
+                for (const sample of buffer) {
+                    sumSquares += sample * sample;
+                }
+                const rms = Math.sqrt(sumSquares / buffer.length);
+                this.emit('userAudioLevel', Math.min(1, rms));
+                this.analyserRafId = requestAnimationFrame(tick);
+            };
+            this.analyserRafId = requestAnimationFrame(tick);
+            logger.debug('[OpenAI] Audio level analyser wired to mic track');
+        }
+        catch (err) {
+            logger.warn('[OpenAI] Audio level analyser setup failed:', err);
+            this.stopAudioLevelAnalysis();
+        }
+    }
+    stopAudioLevelAnalysis() {
+        if (this.analyserRafId !== null) {
+            cancelAnimationFrame(this.analyserRafId);
+            this.analyserRafId = null;
+        }
+        this.analyserNode?.disconnect();
+        this.analyserNode = null;
+        if (this.analyserContext && this.analyserContext.state !== 'closed') {
+            void this.analyserContext.close();
+        }
+        this.analyserContext = null;
+    }
+    setupInternalEventHandlers() {
+        this.client.on('connection_change', (status) => {
+            switch (status) {
+                case 'connected':
+                    this.emit('connectionStateChanged', 'connected');
+                    void this.applyMicNoiseConstraints();
+                    this.startAudioLevelAnalysis();
+                    break;
+                case 'disconnected':
+                    this.oobResponseIds.clear();
+                    this.oobResponseBuffers.clear();
+                    this.stopAudioLevelAnalysis();
+                    this.emit('connectionStateChanged', 'disconnected');
+                    break;
+                case 'connecting':
+                    this.emit('connectionStateChanged', 'connecting');
+                    break;
+            }
+        });
+        this.client.on('audio_done', () => {
+            this.emit('audioDone');
+        });
+        this.client.on('*', (event) => {
+            this.handleServerEvent(event);
+        });
+    }
+    handleServerEvent(event) {
+        const type = event.type;
+        if (!type)
+            return;
+        switch (type) {
+            case 'session.created':
+                this.emit('sessionReady');
+                break;
+            case 'input_audio_buffer.speech_started':
+                this.emit('userSpeechStarted');
+                break;
+            case 'input_audio_buffer.speech_stopped':
+                this.emit('userSpeechStopped');
+                break;
+            case 'input_audio_buffer.committed':
+                this.handleAudioCommitted(event);
+                break;
+            case 'response.created':
+                this.handleResponseCreated(event);
+                break;
+            case 'response.text.delta':
+            case 'response.output_text.delta':
+                this.handleTextDelta(event);
+                break;
+            case 'response.output_audio_transcript.delta':
+                this.emitAssistantDelta(event);
+                break;
+            case 'response.audio_transcript.done':
+            case 'response.output_audio_transcript.done':
+                this.handleAudioTranscriptDone(event);
+                break;
+            case 'response.done':
+            case 'response.completed':
+                this.handleResponseDone(event);
+                break;
+            case 'output_audio_buffer.started':
+                this.sawOutputAudioBufferEvents = true;
+                this.outputAudioPlaying = true;
+                break;
+            case 'output_audio_buffer.stopped':
+            case 'output_audio_buffer.cleared':
+                this.sawOutputAudioBufferEvents = true;
+                this.outputAudioPlaying = false;
+                this.resolvePlaybackEndWaiters(type);
+                break;
+            case 'conversation.item.input_audio_transcription.completed':
+                this.handleNativeTranscriptionCompleted(event);
+                break;
+            case 'response.function_call_arguments.done':
+                this.handleFunctionCallDone(event);
+                break;
+            case 'error':
+                this.handleError(event);
+                break;
+        }
+    }
+    // --- OOB Transcription ---
+    handleAudioCommitted(event) {
+        if (this.transcriptionConfig?.mode !== 'oob')
+            return;
+        const itemId = event.item_id;
+        if (!itemId)
+            return;
+        logger.debug('[OpenAI] OOB: Requesting transcription for item:', itemId);
+        this.client.sendEvent({
+            type: 'response.create',
+            response: {
+                conversation: 'none',
+                output_modalities: ['text'],
+                tools: [],
+                metadata: { purpose: TRANSCRIPTION_PURPOSE },
+                instructions: this.buildOobPrompt(),
+                input: [{ type: 'item_reference', id: itemId }],
+            },
+        });
+    }
+    handleResponseCreated(event) {
+        const response = event.response;
+        const responseId = response?.id;
+        if (!responseId)
+            return;
+        const metadata = response?.metadata;
+        if (metadata?.purpose !== TRANSCRIPTION_PURPOSE)
+            return;
+        this.oobResponseIds.add(responseId);
+        this.oobResponseBuffers.set(responseId, '');
+        logger.debug('[OpenAI] OOB: Registered response:', responseId);
+    }
+    handleTextDelta(event) {
+        const responseId = event.response_id;
+        const delta = event.delta;
+        if (responseId && this.oobResponseIds.has(responseId)) {
+            if (!delta)
+                return;
+            const buffer = (this.oobResponseBuffers.get(responseId) ?? '') + delta;
+            this.oobResponseBuffers.set(responseId, buffer);
+            this.emit('userTranscriptDelta', delta);
+            return;
+        }
+        if (delta) {
+            this.emit('assistantTranscriptDelta', delta);
+        }
+    }
+    handleResponseDone(event) {
+        const response = event.response;
+        const responseId = response?.id;
+        if (responseId && this.oobResponseIds.has(responseId)) {
+            this.finalizeOobTranscription(responseId);
+            return;
+        }
+        this.extractAndEmitUsage(response);
+        const output = response?.output;
+        if (output) {
+            this.processResponseOutput(output);
+        }
+    }
+    extractAndEmitUsage(response) {
+        const usage = response?.usage;
+        if (!usage)
+            return;
+        logger.debug('[OpenAI Realtime] Raw usage data:', JSON.stringify(usage));
+        const inputDetails = usage.input_token_details;
+        const outputDetails = usage.output_token_details;
+        const inputTokens = usage.input_tokens ?? 0;
+        const outputTokens = usage.output_tokens ?? 0;
+        const totalTokens = usage.total_tokens ?? (inputTokens + outputTokens);
+        const cachedDetails = inputDetails?.cached_tokens_details;
+        const textCached = cachedDetails?.text_tokens;
+        const audioCached = cachedDetails?.audio_tokens;
+        const rawTextInput = inputDetails?.text_tokens;
+        const rawAudioInput = inputDetails?.audio_tokens;
+        const providerUsage = {
+            inputTokens,
+            outputTokens,
+            totalTokens,
+            cachedTokens: inputDetails?.cached_tokens,
+            textCachedInputTokens: textCached,
+            audioCachedInputTokens: audioCached,
+            audioInputTokens: rawAudioInput === undefined ? undefined : Math.max(rawAudioInput - (audioCached ?? 0), 0),
+            audioOutputTokens: outputDetails?.audio_tokens,
+            textInputTokens: rawTextInput === undefined ? undefined : Math.max(rawTextInput - (textCached ?? 0), 0),
+            textOutputTokens: outputDetails?.text_tokens,
+        };
+        this.emit('usageUpdated', providerUsage);
+    }
+    processResponseOutput(output) {
+        for (const item of output) {
+            if (item.type === 'function_call' && item.call_id && item.name && item.arguments) {
+                this.emit('functionCall', {
+                    callId: item.call_id,
+                    name: item.name,
+                    arguments: item.arguments,
+                });
+                continue;
+            }
+            if (item.type !== 'message')
+                continue;
+            const content = item.content;
+            if (!content)
+                continue;
+            for (const c of content) {
+                if (c.type === 'text' && c.text) {
+                    this.emit('assistantTranscriptDone', c.text);
+                }
+            }
+        }
+    }
+    finalizeOobTranscription(responseId) {
+        const rawText = (this.oobResponseBuffers.get(responseId) ?? '').trim();
+        this.oobResponseIds.delete(responseId);
+        this.oobResponseBuffers.delete(responseId);
+        if (!rawText)
+            return;
+        const fullText = unwrapTranscript(rawText);
+        if (fullText !== rawText) {
+            logger.debug('[OpenAI] OOB: Unwrapped transcript:', rawText, '→', fullText);
+        }
+        logger.debug('[OpenAI] OOB: Complete:', fullText);
+        this.emit('userTranscriptCompleted', fullText);
+    }
+    buildOobPrompt() {
+        const lang = this.transcriptionConfig?.language ?? 'en';
+        const langHint = lang === 'de'
+            ? 'The user speaks German. Transcribe in German, preserving any English loanwords or technical terms exactly as spoken.'
+            : `The user speaks ${lang}. Transcribe in that language.`;
+        return `# Task: Verbatim Transcription
+
+You are a strict transcription engine. Transcribe exactly what the user said in their most recent spoken turn.
+
+${langHint}
+
+## Rules
+- Transcribe **only** the latest user turn — do not summarize or merge multiple turns.
+- Preserve everything exactly as spoken: filler words (ähm, äh, um), stutters, self-corrections, mispronunciations, slang, incomplete words.
+- Do **not** fix grammar, reorder words, translate, or interpret.
+- Do **not** answer questions, give advice, or continue the conversation.
+
+## Output Format
+Always respond with exactly this JSON structure — no other format, no extra fields:
+
+{"transcript": "<verbatim transcription here>"}
+
+If there is no transcribable content (silence, noise), respond with:
+
+{"transcript": ""}`;
+    }
+    // --- Native Transcription ---
+    handleNativeTranscriptionCompleted(event) {
+        const transcript = event.transcript?.trim();
+        if (!transcript)
+            return;
+        logger.debug('[OpenAI] Native transcription:', transcript);
+        this.emit('userTranscriptCompleted', transcript);
+    }
+    // --- Assistant Events ---
+    emitAssistantDelta(event) {
+        const delta = event.delta;
+        if (delta) {
+            this.emit('assistantTranscriptDelta', delta);
+        }
+    }
+    handleAudioTranscriptDone(event) {
+        const transcript = event.transcript;
+        if (transcript) {
+            this.emit('assistantTranscriptDone', transcript);
+        }
+    }
+    // --- Function Calls ---
+    handleFunctionCallDone(event) {
+        const callId = event.call_id;
+        const name = event.name;
+        const args = event.arguments;
+        if (!callId || !name || !args)
+            return;
+        this.emit('functionCall', { callId, name, arguments: args });
+    }
+    // --- Errors ---
+    handleError(event) {
+        const error = event.error;
+        const code = error?.code;
+        if (code === 'conversation_already_has_active_response') {
+            logger.debug('[OpenAI] Ignoring active response conflict');
+            return;
+        }
+        logger.error('[OpenAI] Server error:', error);
+        this.emit('error', {
+            code,
+            message: error?.message || 'Unknown error',
+        });
+    }
+}

package/dist/services/realtime/openai/openai-tool-mapper.d.ts

@@ -0,0 +1,15 @@
+import type { ToolDefinition } from '../../../types';
+type OpenAIFunctionToolDefinition = {
+    type: 'function';
+    name: string;
+    description: string;
+    parameters: {
+        type: 'object';
+        properties: Record<string, unknown>;
+        required: string[];
+        additionalProperties: boolean;
+    };
+    strict: boolean;
+};
+export declare function mapToolsForOpenAI(tools: ToolDefinition[]): OpenAIFunctionToolDefinition[];
+export {};

package/dist/services/realtime/openai/openai-tool-mapper.js

@@ -0,0 +1,14 @@
+export function mapToolsForOpenAI(tools) {
+    return tools.map((tool) => ({
+        type: 'function',
+        name: tool.name,
+        description: tool.description,
+        parameters: {
+            type: 'object',
+            properties: tool.parameters.properties,
+            required: tool.parameters.required ?? [],
+            additionalProperties: tool.parameters.additionalProperties ?? false,
+        },
+        strict: tool.strict ?? false,
+    }));
+}

package/dist/services/realtime/provider-factory.d.ts

@@ -0,0 +1,4 @@
+import type { RealtimeProviderClient, RealtimeProviderType } from './types';
+export type RealtimeProviderLoader = () => Promise<RealtimeProviderClient> | RealtimeProviderClient;
+export declare function registerRealtimeProvider(providerType: RealtimeProviderType, loader: RealtimeProviderLoader): void;
+export declare function createRealtimeProvider(providerType: RealtimeProviderType): Promise<RealtimeProviderClient>;

package/dist/services/realtime/provider-factory.js

@@ -0,0 +1,12 @@
+const registry = new Map();
+export function registerRealtimeProvider(providerType, loader) {
+    registry.set(providerType, loader);
+}
+export async function createRealtimeProvider(providerType) {
+    const loader = registry.get(providerType);
+    if (!loader) {
+        throw new Error(`No realtime provider registered for "${providerType}". ` +
+            'Import the matching provider module before connecting.');
+    }
+    return loader();
+}