@cogitator-ai/voice 0.1.12 → 0.2.0

package/dist/realtime/gemini-realtime.js

@@ -1,74 +1,149 @@
 import { EventEmitter } from 'node:events';
 import { WebSocket } from 'ws';
-const DEFAULT_MODEL = 'gemini-live-2.5-flash-native-audio';
+const DEFAULT_MODEL = 'gemini-3.8-live';
+const DEFAULT_VOICE = 'Puck';
 const BASE_URL = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
+const CONNECT_TIMEOUT_MS = 30_000;
+const INPUT_MIME_TYPE = 'audio/pcm;rate=16000';
+function rawDataToString(data) {
+    if (Array.isArray(data))
+        return Buffer.concat(data).toString();
+    if (data instanceof ArrayBuffer)
+        return Buffer.from(new Uint8Array(data)).toString();
+    return data.toString();
+}
+/**
+ * Gemini Live API adapter. Input audio is PCM16 mono at 16kHz,
+ * output audio is PCM16 mono at 24kHz.
+ */
 export class GeminiRealtimeAdapter extends EventEmitter {
     config;
     model;
     ws = null;
+    connected = false;
+    modelTurnActive = false;
+    interrupting = false;
+    userTranscript = '';
+    assistantTranscript = '';
+    assistantText = '';
     constructor(config) {
         super();
         this.config = config;
         this.model = config.model ?? DEFAULT_MODEL;
     }
+    get isConnected() {
+        return this.connected;
+    }
     async connect() {
-        const url = `${BASE_URL}?key=${this.config.apiKey}`;
+        if (this.ws) {
+            throw new Error('Already connected or connecting — call close() first');
+        }
         return new Promise((resolve, reject) => {
-            this.ws = new WebSocket(url);
-            this.ws.on('open', () => {
+            let settled = false;
+            const settle = (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                if (err)
+                    reject(err);
+                else
+                    resolve();
+            };
+            const ws = new WebSocket(BASE_URL, {
+                headers: { 'x-goog-api-key': this.config.apiKey },
+            });
+            this.ws = ws;
+            const timer = setTimeout(() => {
+                if (this.ws === ws)
+                    this.ws = null;
+                ws.removeAllListeners();
+                ws.on('error', () => { });
+                ws.terminate();
+                settle(new Error(`Connect timed out after ${CONNECT_TIMEOUT_MS}ms`));
+            }, CONNECT_TIMEOUT_MS);
+            ws.on('open', () => {
                 this.sendSetup();
             });
-            this.ws.on('message', (data) => {
-                const raw = typeof data === 'string' ? data : data.toString();
+            ws.on('message', (data) => {
                 let msg;
                 try {
-                    msg = JSON.parse(raw);
+                    msg = JSON.parse(rawDataToString(data));
                 }
                 catch {
                     this.emit('error', new Error('Failed to parse WebSocket message'));
                     return;
                 }
+                if (msg.error) {
+                    const err = new Error(msg.error.message ?? `Gemini error (code: ${msg.error.code ?? 'unknown'})`);
+                    if (settled)
+                        this.emit('error', err);
+                    else
+                        settle(err);
+                    return;
+                }
                 if (msg.setupComplete) {
+                    this.connected = true;
                     this.emit('connected');
-                    resolve();
+                    settle();
                     return;
                 }
-                this.handleMessage(msg);
+                if (msg.serverContent) {
+                    this.handleServerContent(msg.serverContent);
+                }
+                if (msg.toolCall?.functionCalls?.length) {
+                    void this.handleToolCalls(msg.toolCall.functionCalls);
+                }
+            });
+            ws.on('error', (err) => {
+                if (settled)
+                    this.emit('error', err);
+                else
+                    settle(err);
             });
-            this.ws.on('error', (err) => {
-                this.emit('error', err);
-                reject(err);
+            ws.on('close', (code, reason) => {
+                const reasonStr = reason.toString() || 'connection closed';
+                if (this.ws === ws)
+                    this.ws = null;
+                const wasConnected = this.connected;
+                this.resetTurn();
+                this.connected = false;
+                settle(new Error(`WebSocket closed before connect (code ${code}): ${reasonStr}`));
+                if (wasConnected)
+                    this.emit('disconnected', code, reasonStr);
             });
         });
     }
     pushAudio(chunk) {
         this.send({
             realtimeInput: {
-                mediaChunks: [
-                    {
-                        mimeType: 'audio/pcm;rate=16000',
-                        data: chunk.toString('base64'),
-                    },
-                ],
+                audio: { mimeType: INPUT_MIME_TYPE, data: chunk.toString('base64') },
             },
         });
     }
     sendText(text) {
-        this.send({
-            clientContent: {
-                turns: [{ role: 'user', parts: [{ text }] }],
-                turnComplete: true,
-            },
-        });
+        this.send({ realtimeInput: { text } });
     }
+    /**
+     * Gemini Live barge-in is driven by incoming user audio, not a control message.
+     * While a model turn is in progress this drops the remaining model audio until
+     * the turn completes, simulating an interruption on the consumer side.
+     */
     interrupt() {
-        this.send({
-            clientContent: { turnComplete: true },
-        });
+        if (this.modelTurnActive) {
+            this.interrupting = true;
+        }
     }
     close() {
-        this.ws?.close();
+        const ws = this.ws;
+        this.connected = false;
+        this.resetTurn();
+        if (!ws)
+            return;
         this.ws = null;
+        ws.removeAllListeners();
+        ws.on('error', () => { });
+        ws.close();
     }
     sendSetup() {
         const setup = {
@@ -77,17 +152,15 @@ export class GeminiRealtimeAdapter extends EventEmitter {
                 responseModalities: ['AUDIO'],
                 speechConfig: {
                     voiceConfig: {
-                        prebuiltVoiceConfig: {
-                            voiceName: this.config.voice ?? 'Puck',
-                        },
+                        prebuiltVoiceConfig: { voiceName: this.config.voice ?? DEFAULT_VOICE },
                     },
                 },
             },
+            inputAudioTranscription: {},
+            outputAudioTranscription: {},
         };
         if (this.config.instructions) {
-            setup.systemInstruction = {
-                parts: [{ text: this.config.instructions }],
-            };
+            setup.systemInstruction = { parts: [{ text: this.config.instructions }] };
         }
         if (this.config.tools?.length) {
             setup.tools = [
@@ -102,54 +175,89 @@ export class GeminiRealtimeAdapter extends EventEmitter {
         }
         this.send({ setup });
     }
-    handleMessage(msg) {
-        if (msg.serverContent) {
-            this.handleServerContent(msg.serverContent);
+    handleServerContent(content) {
+        const inputText = content.inputTranscription?.text;
+        if (inputText) {
+            this.userTranscript += inputText;
         }
-        if (msg.toolCall) {
-            void this.handleToolCalls(msg.toolCall.functionCalls);
+        const outputText = content.outputTranscription?.text;
+        const parts = content.modelTurn?.parts ?? [];
+        if (parts.length > 0 || outputText) {
+            this.modelTurnActive = true;
+            this.flushUserTranscript();
         }
-    }
-    handleServerContent(content) {
-        if (!content.modelTurn?.parts)
-            return;
-        for (const part of content.modelTurn.parts) {
-            if (part.inlineData) {
+        if (outputText) {
+            this.assistantTranscript += outputText;
+        }
+        for (const part of parts) {
+            if (part.thought)
+                continue;
+            if (part.inlineData?.data && !this.interrupting) {
                 this.emit('audio', Buffer.from(part.inlineData.data, 'base64'));
             }
             if (part.text) {
-                this.emit('transcript', part.text, 'assistant');
+                this.assistantText += part.text;
             }
         }
+        if (content.interrupted) {
+            this.flushAssistantTranscript();
+            this.modelTurnActive = false;
+            this.interrupting = false;
+            this.emit('speech_start');
+        }
+        if (content.turnComplete) {
+            this.flushUserTranscript();
+            this.flushAssistantTranscript();
+            this.modelTurnActive = false;
+            this.interrupting = false;
+            this.emit('turn_end');
+        }
+    }
+    flushUserTranscript() {
+        const text = this.userTranscript.trim();
+        this.userTranscript = '';
+        if (text)
+            this.emit('transcript', text, 'user');
+    }
+    flushAssistantTranscript() {
+        const text = (this.assistantTranscript || this.assistantText).trim();
+        this.assistantTranscript = '';
+        this.assistantText = '';
+        if (text)
+            this.emit('transcript', text, 'assistant');
+    }
+    resetTurn() {
+        this.modelTurnActive = false;
+        this.interrupting = false;
+        this.userTranscript = '';
+        this.assistantTranscript = '';
+        this.assistantText = '';
     }
     async handleToolCalls(calls) {
-        const responses = await Promise.all(calls.map(async (call) => {
-            this.emit('tool_call', call.name, call.args);
+        const ws = this.ws;
+        const functionResponses = await Promise.all(calls.map(async (call, index) => {
+            const id = call.id ?? `${call.name}-${index}`;
+            const args = call.args ?? {};
+            this.emit('tool_call', call.name, args);
             const tool = this.config.tools?.find((t) => t.name === call.name);
             if (!tool) {
-                return {
-                    id: call.id,
-                    name: call.name,
-                    response: { result: JSON.stringify({ error: `Unknown tool: ${call.name}` }) },
-                };
+                return { id, name: call.name, response: { error: `Unknown tool: ${call.name}` } };
             }
-            let result;
             try {
-                const output = await tool.execute(call.args);
-                result = JSON.stringify(output);
+                const output = await tool.execute(args);
+                return { id, name: call.name, response: { result: output ?? null } };
             }
             catch (err) {
-                result = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
+                return {
+                    id,
+                    name: call.name,
+                    response: { error: err instanceof Error ? err.message : String(err) },
+                };
             }
-            return {
-                id: call.id,
-                name: call.name,
-                response: { result },
-            };
         }));
-        this.send({
-            toolResponse: { functionResponses: responses },
-        });
+        if (this.ws !== ws)
+            return;
+        this.send({ toolResponse: { functionResponses } });
     }
     send(data) {
         const ws = this.ws;

package/dist/realtime/gemini-realtime.js.map

@@ -1 +1 @@
-{"version":3,"file":"gemini-realtime.js","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAC3D,MAAM,QAAQ,GACZ,sHAAsH,CAAC;AAmCzH,MAAM,OAAO,qBAAsB,SAAQ,YAAkC;IAC1D,MAAM,CAAwB;IAC9B,KAAK,CAAS;IACvB,EAAE,GAAqB,IAAI,CAAC;IAEpC,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,GAAG,GAAG,GAAG,QAAQ,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAEpD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;YAE7B,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAqB,EAAE,EAAE;gBAC9C,MAAM,GAAG,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC9D,IAAI,GAAkB,CAAC;gBACvB,IAAI,CAAC;oBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;gBACzC,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;oBACnE,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACvB,OAAO,EAAE,CAAC;oBACV,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBACjC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE;gBACb,WAAW,EAAE;oBACX;wBACE,QAAQ,EAAE,sBAAsB;wBAChC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;qBAC/B;iBACF;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE;gBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC5C,YAAY,EAAE,IAAI;aACnB;SACF,CAAC,CAAC;IACL,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;SACtC,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QACjB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;IACjB,CAAC;IAEO,SAAS;QACf,MAAM,KAAK,GAA4B;YACrC,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;YAC7B,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,CAAC,OAAO,CAAC;gBAC7B,YAAY,EAAE;oBACZ,WAAW,EAAE;wBACX,mBAAmB,EAAE;4BACnB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM;yBACvC;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,CAAC,iBAAiB,GAAG;gBACxB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,KAAK,GAAG;gBACZ;oBACE,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClD,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;wBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;qBACzB,CAAC,CAAC;iBACJ;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACvB,CAAC;IAEO,aAAa,CAAC,GAAkB;QACtC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACtB,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,OAA4B;QACtD,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK;YAAE,OAAO;QAEtC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAA2B;QACvD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO;oBACL,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE;iBAC9E,CAAC;YACJ,CAAC;YAED,IAAI,MAAc,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvF,CAAC;YAED,OAAO;gBACL,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,EAAE,MAAM,EAAE;aACrB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC;YACR,YAAY,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE;SAC/C,CAAC,CAAC;IACL,CAAC;IAEO,IAAI,CAAC,IAA6B;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,EAAE,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"gemini-realtime.js","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,MAAM,QAAQ,GACZ,sHAAsH,CAAC;AACzH,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,eAAe,GAAG,sBAAsB,CAAC;AAwC/C,SAAS,eAAe,CAAC,IAAuB;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC/D,IAAI,IAAI,YAAY,WAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACrF,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,qBAAsB,SAAQ,YAAkC;IAC1D,MAAM,CAAwB;IAC9B,KAAK,CAAS;IACvB,EAAE,GAAqB,IAAI,CAAC;IAC5B,SAAS,GAAG,KAAK,CAAC;IAClB,eAAe,GAAG,KAAK,CAAC;IACxB,YAAY,GAAG,KAAK,CAAC;IACrB,cAAc,GAAG,EAAE,CAAC;IACpB,mBAAmB,GAAG,EAAE,CAAC;IACzB,aAAa,GAAG,EAAE,CAAC;IAE3B,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,EAAE;gBAC7B,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,EAAE,CAAC;YACjB,CAAC,CAAC;YAEF,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;aAClD,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAEb,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACnC,EAAE,CAAC,kBAAkB,EAAE,CAAC;gBACxB,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACzB,EAAE,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,kBAAkB,IAAI,CAAC,CAAC,CAAC;YACvE,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEvB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAuB,EAAE,EAAE;gBAC3C,IAAI,GAAkB,CAAC;gBACvB,IAAI,CAAC;oBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAkB,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;oBACnE,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACd,MAAM,GAAG,GAAG,IAAI,KAAK,CACnB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,uBAAuB,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,SAAS,GAAG,CAC3E,CAAC;oBACF,IAAI,OAAO;wBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;;wBAChC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACjB,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;oBACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACvB,MAAM,EAAE,CAAC;oBACT,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;oBACtB,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBAC9C,CAAC;gBAED,IAAI,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;oBACxC,KAAK,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC5B,IAAI,OAAO;oBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;;oBAChC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;gBAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,mBAAmB,CAAC;gBAC3D,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACnC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;gBACpC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBACvB,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC;gBAClF,IAAI,YAAY;oBAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE;gBACb,KAAK,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;aACrE;SACF,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,SAAS;QACP,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK;QACH,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACxB,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzB,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAEO,SAAS;QACf,MAAM,KAAK,GAA4B;YACrC,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;YAC7B,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,CAAC,OAAO,CAAC;gBAC7B,YAAY,EAAE;oBACZ,WAAW,EAAE;wBACX,mBAAmB,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,aAAa,EAAE;qBACvE;iBACF;aACF;YACD,uBAAuB,EAAE,EAAE;YAC3B,wBAAwB,EAAE,EAAE;SAC7B,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,CAAC,iBAAiB,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,KAAK,GAAG;gBACZ;oBACE,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClD,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;wBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;qBACzB,CAAC,CAAC;iBACJ;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACvB,CAAC;IAEO,mBAAmB,CAAC,OAA4B;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC;QACnD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,cAAc,IAAI,SAAS,CAAC;QACnC,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,mBAAmB,IAAI,UAAU,CAAC;QACzC,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,OAAO;gBAAE,SAAS;YAC3B,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,mBAAmB;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,wBAAwB;QAC9B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IACvD,CAAC;IAEO,SAAS;QACf,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAA2B;QACvD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CACzC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAExC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,iBAAiB,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;YACpF,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACxC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YACvE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,EAAE;oBACF,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;iBACtE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;YAAE,OAAO;QAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACrD,CAAC;IAEO,IAAI,CAAC,IAA6B;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,EAAE,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}

package/dist/realtime/openai-realtime.d.ts

@@ -2,17 +2,26 @@ import { EventEmitter } from 'node:events';
 import type { RealtimeSessionConfig } from '../types.js';
 interface OpenAIRealtimeEvents {
     connected: [];
+    disconnected: [code: number, reason: string];
     audio: [chunk: Buffer];
     transcript: [text: string, role: 'user' | 'assistant'];
     tool_call: [name: string, args: unknown];
     speech_start: [];
+    turn_end: [];
     error: [error: Error];
 }
+/**
+ * OpenAI Realtime API (GA interface) adapter. Audio in both directions is
+ * PCM16 mono at 24kHz.
+ */
 export declare class OpenAIRealtimeAdapter extends EventEmitter<OpenAIRealtimeEvents> {
     private readonly config;
     private readonly model;
     private ws;
+    private connected;
+    private responseActive;
     constructor(config: RealtimeSessionConfig);
+    get isConnected(): boolean;
     connect(): Promise<void>;
     pushAudio(chunk: Buffer): void;
     sendText(text: string): void;
@@ -20,7 +29,9 @@ export declare class OpenAIRealtimeAdapter extends EventEmitter<OpenAIRealtimeEv
     close(): void;
     private sendSessionUpdate;
     private handleMessage;
-    private handleToolCall;
+    private handleResponseDone;
+    private handleToolCalls;
+    private executeToolCall;
     private send;
 }
 export {};

package/dist/realtime/openai-realtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openai-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAKzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;gBAExB,MAAM,EAAE,qBAAqB;IAMnC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAO9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAY5B,SAAS,IAAI,IAAI;IAIjB,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,iBAAiB;IAwBzB,OAAO,CAAC,aAAa;YAuCP,cAAc;IAmC5B,OAAO,CAAC,IAAI;CAMb"}
+{"version":3,"file":"openai-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AASzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAkBD;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,qBAAqB;IAMzC,IAAI,WAAW,IAAI,OAAO,CAEzB;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA6D9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAO9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAY5B,SAAS,IAAI,IAAI;IAKjB,KAAK,IAAI,IAAI;IAWb,OAAO,CAAC,iBAAiB;IAkCzB,OAAO,CAAC,aAAa;IA8DrB,OAAO,CAAC,kBAAkB;YA2BZ,eAAe;YAcf,eAAe;IAkC7B,OAAO,CAAC,IAAI;CAMb"}