@openai/agents-realtime 0.0.1

package/dist/openaiRealtimeWebsocket.js

@@ -0,0 +1,293 @@
+import { isBrowserEnvironment, WebSocket, } from '@openai/agents-realtime/_shims';
+import { OpenAIRealtimeBase, } from "./openaiRealtimeBase.js";
+import { base64ToArrayBuffer, HEADERS, WEBSOCKET_META } from "./utils.js";
+import { UserError } from '@openai/agents-core';
+import { parseRealtimeEvent } from "./openaiRealtimeEvents.js";
+/**
+ * Transport layer that's handling the connection between the client and OpenAI's Realtime API
+ * via WebSockets. While this transport layer is designed to be used within a RealtimeSession, it
+ * can also be used standalone if you want to have a direct connection to the Realtime API.
+ */
+export class OpenAIRealtimeWebSocket extends OpenAIRealtimeBase {
+    #apiKey;
+    #url;
+    #state = {
+        status: 'disconnected',
+        websocket: undefined,
+    };
+    #useInsecureApiKey;
+    #currentItemId;
+    #currentAudioContentIndex;
+    /**
+     * Timestamp maintained by the transport layer to aid with the calculation of the elapsed time
+     * since the response started to compute the right interruption time.
+     *
+     * Mostly internal but might be used by extended transport layers for their interruption
+     * calculation.
+     */
+    _firstAudioTimestamp;
+    _audioLengthMs = 0;
+    #ongoingResponse = false;
+    constructor(options = {}) {
+        super(options);
+        this.#url = `wss://api.openai.com/v1/realtime?model=${this.currentModel}`;
+        this.#useInsecureApiKey = options.useInsecureApiKey ?? false;
+    }
+    /**
+     * The current status of the WebSocket connection.
+     */
+    get status() {
+        return this.#state.status;
+    }
+    /**
+     * The current connection state of the WebSocket connection.
+     */
+    get connectionState() {
+        return this.#state;
+    }
+    /**
+     * Always returns `null` as the WebSocket transport layer does not handle muting. Instead,
+     * this should be handled by the client by not triggering the `sendAudio` method.
+     */
+    get muted() {
+        return null;
+    }
+    /**
+     * The current item ID of the ongoing response.
+     */
+    get currentItemId() {
+        return this.#currentItemId;
+    }
+    /**
+     * Triggers the `audio` event that a client might listen to to receive the audio buffer.
+     * Protected for you to be able to override and disable emitting this event in case your extended
+     * transport layer handles audio internally.
+     *
+     * @param audioEvent - The audio event to emit.
+     */
+    _onAudio(audioEvent) {
+        this.emit('audio', audioEvent);
+    }
+    #setupWebSocket(resolve, reject, sessionConfig) {
+        if (this.#state.websocket) {
+            resolve();
+            return;
+        }
+        if (!this.#apiKey) {
+            throw new UserError('API key is not set. Please call `connect()` with an API key first.');
+        }
+        if (isBrowserEnvironment() &&
+            !this.#apiKey.startsWith('ek_') &&
+            !this.#useInsecureApiKey) {
+            throw new UserError('Using the WebSocket connection in a browser environment requires an ephemeral client key. If you have to use a regular API key, set the `useInsecureApiKey` option to true.');
+        }
+        const websocketArguments = isBrowserEnvironment()
+            ? [
+                'realtime',
+                // Auth
+                'openai-insecure-api-key.' + this.#apiKey,
+                // Beta protocol, required
+                'openai-beta.realtime-v1',
+                // Version header
+                WEBSOCKET_META,
+            ]
+            : {
+                headers: {
+                    Authorization: `Bearer ${this.#apiKey}`,
+                    'OpenAI-Beta': 'realtime=v1',
+                    ...HEADERS,
+                },
+            };
+        const ws = new WebSocket(this.#url, websocketArguments);
+        this.#state = {
+            status: 'connecting',
+            websocket: ws,
+        };
+        this.emit('connection_change', this.#state.status);
+        ws.addEventListener('open', () => {
+            this.#state = {
+                status: 'connected',
+                websocket: ws,
+            };
+            this.emit('connection_change', this.#state.status);
+            this._onOpen();
+            resolve();
+        });
+        ws.addEventListener('error', (error) => {
+            this._onError(error);
+            this.#state = {
+                status: 'disconnected',
+                websocket: undefined,
+            };
+            this.emit('connection_change', this.#state.status);
+            reject(error);
+        });
+        ws.addEventListener('message', (message) => {
+            this._onMessage(message);
+            const { data: parsed, isGeneric } = parseRealtimeEvent(message);
+            if (!parsed || isGeneric) {
+                return;
+            }
+            if (parsed.type === 'response.audio.delta') {
+                this.#currentAudioContentIndex = parsed.content_index;
+                this.#currentItemId = parsed.item_id;
+                if (this._firstAudioTimestamp === undefined) {
+                    // If the response start timestamp is not set, we set it to the current time.
+                    // This is used to calculate the elapsed time for interruption.
+                    this._firstAudioTimestamp = Date.now();
+                    this._audioLengthMs = 0;
+                }
+                const buff = base64ToArrayBuffer(parsed.delta);
+                // calculate the audio length in milliseconds assuming 24kHz pcm16le
+                this._audioLengthMs += buff.byteLength / 24 / 2; // 24kHz * 2 bytes per sample
+                const audioEvent = {
+                    type: 'audio',
+                    data: buff,
+                    responseId: parsed.response_id,
+                };
+                this._onAudio(audioEvent);
+            }
+            else if (parsed.type === 'input_audio_buffer.speech_started') {
+                this.interrupt();
+            }
+            else if (parsed.type === 'response.created') {
+                this.#ongoingResponse = true;
+            }
+            else if (parsed.type === 'response.done') {
+                this.#ongoingResponse = false;
+            }
+            else if (parsed.type === 'session.created') {
+                this._tracingConfig = parsed.session.tracing;
+                // Trying to turn on tracing after the session is created
+                this._updateTracingConfig(sessionConfig.tracing ?? 'auto');
+            }
+        });
+        ws.addEventListener('close', () => {
+            this.#state = {
+                status: 'disconnected',
+                websocket: undefined,
+            };
+            this.emit('connection_change', this.#state.status);
+            this._onClose();
+        });
+    }
+    async connect(options) {
+        const model = options.model ?? this.currentModel;
+        this.currentModel = model;
+        this.#apiKey = await this._getApiKey(options);
+        this.#url =
+            options.url ??
+                `wss://api.openai.com/v1/realtime?model=${this.currentModel}`;
+        const sessionConfig = {
+            ...(options.initialSessionConfig || {}),
+            model: this.currentModel,
+        };
+        await new Promise((resolve, reject) => {
+            try {
+                this.#setupWebSocket(resolve, reject, sessionConfig);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+        await this.updateSessionConfig(sessionConfig);
+    }
+    /**
+     * Send an event to the Realtime API. This will stringify the event and send it directly to the
+     * API. This can be used if you want to take control over the connection and send events manually.
+     *
+     * @param event - The event to send.
+     */
+    sendEvent(event) {
+        if (!this.#state.websocket) {
+            throw new Error('WebSocket is not connected. Make sure you call `connect()` before sending events.');
+        }
+        this.#state.websocket.send(JSON.stringify(event));
+    }
+    /**
+     * Close the WebSocket connection.
+     *
+     * This will also reset any internal connection tracking used for interruption handling.
+     */
+    close() {
+        this.#state.websocket?.close();
+        this.#currentItemId = undefined;
+        this._firstAudioTimestamp = undefined;
+        this._audioLengthMs = 0;
+        this.#currentAudioContentIndex = undefined;
+    }
+    /**
+     * Will throw an error as the WebSocket transport layer does not support muting.
+     */
+    mute(_muted) {
+        throw new Error('Mute is not supported for the WebSocket transport. You have to mute the audio input yourself.');
+    }
+    /**
+     * Send an audio buffer to the Realtime API. This is used for your client to send audio to the
+     * model to respond.
+     *
+     * @param audio - The audio buffer to send.
+     * @param options - The options for the audio buffer.
+     */
+    sendAudio(audio, options = {}) {
+        if (this.#state.status === 'connected') {
+            super.sendAudio(audio, options);
+        }
+    }
+    /**
+     * Send a cancel response event to the Realtime API. This is used to cancel an ongoing
+     *  response that the model is currently generating.
+     */
+    _cancelResponse() {
+        // cancel the ongoing response
+        if (this.#ongoingResponse) {
+            this.sendEvent({
+                type: 'response.cancel',
+            });
+        }
+    }
+    /**
+     * Do NOT call this method directly. Call `interrupt()` instead for proper interruption handling.
+     *
+     * This method is used to send the right events to the API to inform the model that the user has
+     * interrupted the response. It might be overridden/extended by an extended transport layer. See
+     * the `TwilioRealtimeTransportLayer` for an example.
+     *
+     * @param elapsedTime - The elapsed time since the response started.
+     */
+    _interrupt(elapsedTime) {
+        // immediately emit this event so the client can stop playing audio
+        this.emit('audio_interrupted');
+        this.sendEvent({
+            type: 'conversation.item.truncate',
+            item_id: this.#currentItemId,
+            content_index: this.#currentAudioContentIndex,
+            audio_end_ms: elapsedTime,
+        });
+    }
+    /**
+     * Interrupt the ongoing response. This method is triggered automatically by the client when
+     * voice activity detection (VAD) is enabled (default) as well as when an output guardrail got
+     * triggered.
+     *
+     * You can also call this method directly if you want to interrupt the conversation for example
+     * based on an event in the client.
+     */
+    interrupt() {
+        if (!this.#currentItemId || typeof this._firstAudioTimestamp !== 'number') {
+            return;
+        }
+        this._cancelResponse();
+        const elapsedTime = Date.now() - this._firstAudioTimestamp;
+        console.log(`Interrupting response after ${elapsedTime}ms`);
+        console.log(`Audio length: ${this._audioLengthMs}ms`);
+        if (elapsedTime >= 0 && elapsedTime < this._audioLengthMs) {
+            this._interrupt(elapsedTime);
+        }
+        this.#currentItemId = undefined;
+        this._firstAudioTimestamp = undefined;
+        this._audioLengthMs = 0;
+        this.#currentAudioContentIndex = undefined;
+    }
+}
+//# sourceMappingURL=openaiRealtimeWebsocket.js.map

package/dist/openaiRealtimeWebsocket.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaiRealtimeWebsocket.js","sourceRoot":"","sources":["../src/openaiRealtimeWebsocket.ts"],"names":[],"mappings":"OAAO,EACL,oBAAoB,EACpB,SAAS,GACV,MAAM,gCAAgC;OAOhC,EACL,kBAAkB,GAEnB;OACM,EAAE,mBAAmB,EAAE,OAAO,EAAE,cAAc,EAAE;OAChD,EAAE,SAAS,EAAE,MAAM,qBAAqB;OAExC,EAAE,kBAAkB,EAAE;AAiC7B;;;;GAIG;AACH,MAAM,OAAO,uBACX,SAAQ,kBAAkB;IAG1B,OAAO,CAAqB;IAC5B,IAAI,CAAS;IACb,MAAM,GAAmB;QACvB,MAAM,EAAE,cAAc;QACtB,SAAS,EAAE,SAAS;KACrB,CAAC;IACF,kBAAkB,CAAU;IAC5B,cAAc,CAAqB;IACnC,yBAAyB,CAAqB;IAC9C;;;;;;OAMG;IACO,oBAAoB,CAAqB;IACzC,cAAc,GAAW,CAAC,CAAC;IACrC,gBAAgB,GAAY,KAAK,CAAC;IAElC,YAAY,UAA0C,EAAE;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,0CAA0C,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1E,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAc,aAAa;QACzB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACO,QAAQ,CAAC,UAA+B;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,eAAe,CACb,OAA8B,EAC9B,MAA8B,EAC9B,aAA6C;QAE7C,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,SAAS,CACjB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,IACE,oBAAoB,EAAE;YACtB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAC/B,CAAC,IAAI,CAAC,kBAAkB,EACxB,CAAC;YACD,MAAM,IAAI,SAAS,CACjB,6KAA6K,CAC9K,CAAC;QACJ,CAAC;QAED,MAAM,kBAAkB,GAAG,oBAAoB,EAAE;YAC/C,CAAC,CAAC;gBACE,UAAU;gBACV,OAAO;gBACP,0BAA0B,GAAG,IAAI,CAAC,OAAO;gBACzC,0BAA0B;gBAC1B,yBAAyB;gBACzB,iBAAiB;gBACjB,cAAc;aACf;YACH,CAAC,CAAC;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE;oBACvC,aAAa,EAAE,aAAa;oBAC5B,GAAG,OAAO;iBACX;aACF,CAAC;QAEN,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAyB,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,YAAY;YACpB,SAAS,EAAE,EAAE;SACd,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEnD,EAAE,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,EAAE;aACd,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS;aACrB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACzC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACzB,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC3C,IAAI,CAAC,yBAAyB,GAAG,MAAM,CAAC,aAAa,CAAC;gBACtD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;gBACrC,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;oBAC5C,6EAA6E;oBAC7E,+DAA+D;oBAC/D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;gBAC1B,CAAC;gBAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC/C,oEAAoE;gBACpE,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;gBAE9E,MAAM,UAAU,GAAwB;oBACtC,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,MAAM,CAAC,WAAW;iBAC/B,CAAC;gBACF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC5B,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,mCAAmC,EAAE,CAAC;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC3C,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChC,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBAC7C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC7C,yDAAyD;gBACzD,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAChC,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS;aACrB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA6C;QACzD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;QACjD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI;YACP,OAAO,CAAC,GAAG;gBACX,0CAA0C,IAAI,CAAC,YAAY,EAAE,CAAC;QAEhE,MAAM,aAAa,GAAmC;YACpD,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,EAAE,CAAC;YACvC,KAAK,EAAE,IAAI,CAAC,YAAY;SACzB,CAAC;QAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,IAAI,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;YACvD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,KAA4B;QACpC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,yBAAyB,GAAG,SAAS,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,MAAe;QAClB,MAAM,IAAI,KAAK,CACb,+FAA+F,CAChG,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,SAAS,CAAC,KAAkB,EAAE,UAAgC,EAAE;QAC9D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACvC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,8BAA8B;QAC9B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,iBAAiB;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,WAAmB;QAC5B,mEAAmE;QACnE,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,4BAA4B;YAClC,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,aAAa,EAAE,IAAI,CAAC,yBAAyB;YAC7C,YAAY,EAAE,WAAW;SAC1B,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,QAAQ,EAAE,CAAC;YAC1E,OAAO;QACT,CAAC;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,+BAA+B,WAAW,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACtD,IAAI,WAAW,IAAI,CAAC,IAAI,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC1D,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,yBAAyB,GAAG,SAAS,CAAC;IAC7C,CAAC;CACF"}

package/dist/openaiRealtimeWebsocket.mjs

@@ -0,0 +1,293 @@
+import { isBrowserEnvironment, WebSocket, } from '@openai/agents-realtime/_shims';
+import { OpenAIRealtimeBase, } from "./openaiRealtimeBase.mjs";
+import { base64ToArrayBuffer, HEADERS, WEBSOCKET_META } from "./utils.mjs";
+import { UserError } from '@openai/agents-core';
+import { parseRealtimeEvent } from "./openaiRealtimeEvents.mjs";
+/**
+ * Transport layer that's handling the connection between the client and OpenAI's Realtime API
+ * via WebSockets. While this transport layer is designed to be used within a RealtimeSession, it
+ * can also be used standalone if you want to have a direct connection to the Realtime API.
+ */
+export class OpenAIRealtimeWebSocket extends OpenAIRealtimeBase {
+    #apiKey;
+    #url;
+    #state = {
+        status: 'disconnected',
+        websocket: undefined,
+    };
+    #useInsecureApiKey;
+    #currentItemId;
+    #currentAudioContentIndex;
+    /**
+     * Timestamp maintained by the transport layer to aid with the calculation of the elapsed time
+     * since the response started to compute the right interruption time.
+     *
+     * Mostly internal but might be used by extended transport layers for their interruption
+     * calculation.
+     */
+    _firstAudioTimestamp;
+    _audioLengthMs = 0;
+    #ongoingResponse = false;
+    constructor(options = {}) {
+        super(options);
+        this.#url = `wss://api.openai.com/v1/realtime?model=${this.currentModel}`;
+        this.#useInsecureApiKey = options.useInsecureApiKey ?? false;
+    }
+    /**
+     * The current status of the WebSocket connection.
+     */
+    get status() {
+        return this.#state.status;
+    }
+    /**
+     * The current connection state of the WebSocket connection.
+     */
+    get connectionState() {
+        return this.#state;
+    }
+    /**
+     * Always returns `null` as the WebSocket transport layer does not handle muting. Instead,
+     * this should be handled by the client by not triggering the `sendAudio` method.
+     */
+    get muted() {
+        return null;
+    }
+    /**
+     * The current item ID of the ongoing response.
+     */
+    get currentItemId() {
+        return this.#currentItemId;
+    }
+    /**
+     * Triggers the `audio` event that a client might listen to to receive the audio buffer.
+     * Protected for you to be able to override and disable emitting this event in case your extended
+     * transport layer handles audio internally.
+     *
+     * @param audioEvent - The audio event to emit.
+     */
+    _onAudio(audioEvent) {
+        this.emit('audio', audioEvent);
+    }
+    #setupWebSocket(resolve, reject, sessionConfig) {
+        if (this.#state.websocket) {
+            resolve();
+            return;
+        }
+        if (!this.#apiKey) {
+            throw new UserError('API key is not set. Please call `connect()` with an API key first.');
+        }
+        if (isBrowserEnvironment() &&
+            !this.#apiKey.startsWith('ek_') &&
+            !this.#useInsecureApiKey) {
+            throw new UserError('Using the WebSocket connection in a browser environment requires an ephemeral client key. If you have to use a regular API key, set the `useInsecureApiKey` option to true.');
+        }
+        const websocketArguments = isBrowserEnvironment()
+            ? [
+                'realtime',
+                // Auth
+                'openai-insecure-api-key.' + this.#apiKey,
+                // Beta protocol, required
+                'openai-beta.realtime-v1',
+                // Version header
+                WEBSOCKET_META,
+            ]
+            : {
+                headers: {
+                    Authorization: `Bearer ${this.#apiKey}`,
+                    'OpenAI-Beta': 'realtime=v1',
+                    ...HEADERS,
+                },
+            };
+        const ws = new WebSocket(this.#url, websocketArguments);
+        this.#state = {
+            status: 'connecting',
+            websocket: ws,
+        };
+        this.emit('connection_change', this.#state.status);
+        ws.addEventListener('open', () => {
+            this.#state = {
+                status: 'connected',
+                websocket: ws,
+            };
+            this.emit('connection_change', this.#state.status);
+            this._onOpen();
+            resolve();
+        });
+        ws.addEventListener('error', (error) => {
+            this._onError(error);
+            this.#state = {
+                status: 'disconnected',
+                websocket: undefined,
+            };
+            this.emit('connection_change', this.#state.status);
+            reject(error);
+        });
+        ws.addEventListener('message', (message) => {
+            this._onMessage(message);
+            const { data: parsed, isGeneric } = parseRealtimeEvent(message);
+            if (!parsed || isGeneric) {
+                return;
+            }
+            if (parsed.type === 'response.audio.delta') {
+                this.#currentAudioContentIndex = parsed.content_index;
+                this.#currentItemId = parsed.item_id;
+                if (this._firstAudioTimestamp === undefined) {
+                    // If the response start timestamp is not set, we set it to the current time.
+                    // This is used to calculate the elapsed time for interruption.
+                    this._firstAudioTimestamp = Date.now();
+                    this._audioLengthMs = 0;
+                }
+                const buff = base64ToArrayBuffer(parsed.delta);
+                // calculate the audio length in milliseconds assuming 24kHz pcm16le
+                this._audioLengthMs += buff.byteLength / 24 / 2; // 24kHz * 2 bytes per sample
+                const audioEvent = {
+                    type: 'audio',
+                    data: buff,
+                    responseId: parsed.response_id,
+                };
+                this._onAudio(audioEvent);
+            }
+            else if (parsed.type === 'input_audio_buffer.speech_started') {
+                this.interrupt();
+            }
+            else if (parsed.type === 'response.created') {
+                this.#ongoingResponse = true;
+            }
+            else if (parsed.type === 'response.done') {
+                this.#ongoingResponse = false;
+            }
+            else if (parsed.type === 'session.created') {
+                this._tracingConfig = parsed.session.tracing;
+                // Trying to turn on tracing after the session is created
+                this._updateTracingConfig(sessionConfig.tracing ?? 'auto');
+            }
+        });
+        ws.addEventListener('close', () => {
+            this.#state = {
+                status: 'disconnected',
+                websocket: undefined,
+            };
+            this.emit('connection_change', this.#state.status);
+            this._onClose();
+        });
+    }
+    async connect(options) {
+        const model = options.model ?? this.currentModel;
+        this.currentModel = model;
+        this.#apiKey = await this._getApiKey(options);
+        this.#url =
+            options.url ??
+                `wss://api.openai.com/v1/realtime?model=${this.currentModel}`;
+        const sessionConfig = {
+            ...(options.initialSessionConfig || {}),
+            model: this.currentModel,
+        };
+        await new Promise((resolve, reject) => {
+            try {
+                this.#setupWebSocket(resolve, reject, sessionConfig);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+        await this.updateSessionConfig(sessionConfig);
+    }
+    /**
+     * Send an event to the Realtime API. This will stringify the event and send it directly to the
+     * API. This can be used if you want to take control over the connection and send events manually.
+     *
+     * @param event - The event to send.
+     */
+    sendEvent(event) {
+        if (!this.#state.websocket) {
+            throw new Error('WebSocket is not connected. Make sure you call `connect()` before sending events.');
+        }
+        this.#state.websocket.send(JSON.stringify(event));
+    }
+    /**
+     * Close the WebSocket connection.
+     *
+     * This will also reset any internal connection tracking used for interruption handling.
+     */
+    close() {
+        this.#state.websocket?.close();
+        this.#currentItemId = undefined;
+        this._firstAudioTimestamp = undefined;
+        this._audioLengthMs = 0;
+        this.#currentAudioContentIndex = undefined;
+    }
+    /**
+     * Will throw an error as the WebSocket transport layer does not support muting.
+     */
+    mute(_muted) {
+        throw new Error('Mute is not supported for the WebSocket transport. You have to mute the audio input yourself.');
+    }
+    /**
+     * Send an audio buffer to the Realtime API. This is used for your client to send audio to the
+     * model to respond.
+     *
+     * @param audio - The audio buffer to send.
+     * @param options - The options for the audio buffer.
+     */
+    sendAudio(audio, options = {}) {
+        if (this.#state.status === 'connected') {
+            super.sendAudio(audio, options);
+        }
+    }
+    /**
+     * Send a cancel response event to the Realtime API. This is used to cancel an ongoing
+     *  response that the model is currently generating.
+     */
+    _cancelResponse() {
+        // cancel the ongoing response
+        if (this.#ongoingResponse) {
+            this.sendEvent({
+                type: 'response.cancel',
+            });
+        }
+    }
+    /**
+     * Do NOT call this method directly. Call `interrupt()` instead for proper interruption handling.
+     *
+     * This method is used to send the right events to the API to inform the model that the user has
+     * interrupted the response. It might be overridden/extended by an extended transport layer. See
+     * the `TwilioRealtimeTransportLayer` for an example.
+     *
+     * @param elapsedTime - The elapsed time since the response started.
+     */
+    _interrupt(elapsedTime) {
+        // immediately emit this event so the client can stop playing audio
+        this.emit('audio_interrupted');
+        this.sendEvent({
+            type: 'conversation.item.truncate',
+            item_id: this.#currentItemId,
+            content_index: this.#currentAudioContentIndex,
+            audio_end_ms: elapsedTime,
+        });
+    }
+    /**
+     * Interrupt the ongoing response. This method is triggered automatically by the client when
+     * voice activity detection (VAD) is enabled (default) as well as when an output guardrail got
+     * triggered.
+     *
+     * You can also call this method directly if you want to interrupt the conversation for example
+     * based on an event in the client.
+     */
+    interrupt() {
+        if (!this.#currentItemId || typeof this._firstAudioTimestamp !== 'number') {
+            return;
+        }
+        this._cancelResponse();
+        const elapsedTime = Date.now() - this._firstAudioTimestamp;
+        console.log(`Interrupting response after ${elapsedTime}ms`);
+        console.log(`Audio length: ${this._audioLengthMs}ms`);
+        if (elapsedTime >= 0 && elapsedTime < this._audioLengthMs) {
+            this._interrupt(elapsedTime);
+        }
+        this.#currentItemId = undefined;
+        this._firstAudioTimestamp = undefined;
+        this._audioLengthMs = 0;
+        this.#currentAudioContentIndex = undefined;
+    }
+}
+//# sourceMappingURL=openaiRealtimeWebsocket.mjs.map

package/dist/openaiRealtimeWebsocket.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaiRealtimeWebsocket.mjs","sourceRoot":"","sources":["../src/openaiRealtimeWebsocket.ts"],"names":[],"mappings":"OAAO,EACL,oBAAoB,EACpB,SAAS,GACV,MAAM,gCAAgC;OAOhC,EACL,kBAAkB,GAEnB;OACM,EAAE,mBAAmB,EAAE,OAAO,EAAE,cAAc,EAAE;OAChD,EAAE,SAAS,EAAE,MAAM,qBAAqB;OAExC,EAAE,kBAAkB,EAAE;AAiC7B;;;;GAIG;AACH,MAAM,OAAO,uBACX,SAAQ,kBAAkB;IAG1B,OAAO,CAAqB;IAC5B,IAAI,CAAS;IACb,MAAM,GAAmB;QACvB,MAAM,EAAE,cAAc;QACtB,SAAS,EAAE,SAAS;KACrB,CAAC;IACF,kBAAkB,CAAU;IAC5B,cAAc,CAAqB;IACnC,yBAAyB,CAAqB;IAC9C;;;;;;OAMG;IACO,oBAAoB,CAAqB;IACzC,cAAc,GAAW,CAAC,CAAC;IACrC,gBAAgB,GAAY,KAAK,CAAC;IAElC,YAAY,UAA0C,EAAE;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,0CAA0C,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1E,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAc,aAAa;QACzB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACO,QAAQ,CAAC,UAA+B;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,eAAe,CACb,OAA8B,EAC9B,MAA8B,EAC9B,aAA6C;QAE7C,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,SAAS,CACjB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,IACE,oBAAoB,EAAE;YACtB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAC/B,CAAC,IAAI,CAAC,kBAAkB,EACxB,CAAC;YACD,MAAM,IAAI,SAAS,CACjB,6KAA6K,CAC9K,CAAC;QACJ,CAAC;QAED,MAAM,kBAAkB,GAAG,oBAAoB,EAAE;YAC/C,CAAC,CAAC;gBACE,UAAU;gBACV,OAAO;gBACP,0BAA0B,GAAG,IAAI,CAAC,OAAO;gBACzC,0BAA0B;gBAC1B,yBAAyB;gBACzB,iBAAiB;gBACjB,cAAc;aACf;YACH,CAAC,CAAC;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE;oBACvC,aAAa,EAAE,aAAa;oBAC5B,GAAG,OAAO;iBACX;aACF,CAAC;QAEN,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAyB,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,YAAY;YACpB,SAAS,EAAE,EAAE;SACd,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEnD,EAAE,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,EAAE;aACd,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS;aACrB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACzC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACzB,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC3C,IAAI,CAAC,yBAAyB,GAAG,MAAM,CAAC,aAAa,CAAC;gBACtD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;gBACrC,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;oBAC5C,6EAA6E;oBAC7E,+DAA+D;oBAC/D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;gBAC1B,CAAC;gBAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC/C,oEAAoE;gBACpE,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;gBAE9E,MAAM,UAAU,GAAwB;oBACtC,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,MAAM,CAAC,WAAW;iBAC/B,CAAC;gBACF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC5B,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,mCAAmC,EAAE,CAAC;gBAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC3C,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChC,CAAC;iBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBAC7C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC7C,yDAAyD;gBACzD,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAChC,IAAI,CAAC,MAAM,GAAG;gBACZ,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS;aACrB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA6C;QACzD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;QACjD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI;YACP,OAAO,CAAC,GAAG;gBACX,0CAA0C,IAAI,CAAC,YAAY,EAAE,CAAC;QAEhE,MAAM,aAAa,GAAmC;YACpD,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,EAAE,CAAC;YACvC,KAAK,EAAE,IAAI,CAAC,YAAY;SACzB,CAAC;QAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,IAAI,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;YACvD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,KAA4B;QACpC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,yBAAyB,GAAG,SAAS,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,MAAe;QAClB,MAAM,IAAI,KAAK,CACb,+FAA+F,CAChG,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,SAAS,CAAC,KAAkB,EAAE,UAAgC,EAAE;QAC9D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACvC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,8BAA8B;QAC9B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,iBAAiB;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,WAAmB;QAC5B,mEAAmE;QACnE,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,4BAA4B;YAClC,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,aAAa,EAAE,IAAI,CAAC,yBAAyB;YAC7C,YAAY,EAAE,WAAW;SAC1B,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,QAAQ,EAAE,CAAC;YAC1E,OAAO;QACT,CAAC;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,+BAA+B,WAAW,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACtD,IAAI,WAAW,IAAI,CAAC,IAAI,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC1D,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,yBAAyB,GAAG,SAAS,CAAC;IAC7C,CAAC;CACF"}

package/dist/realtimeAgent.d.ts

@@ -0,0 +1,49 @@
+import { Agent, type AgentConfiguration, Handoff, TextOutput, UnknownContext } from '@openai/agents-core';
+import { RealtimeContextData } from './realtimeSession';
+export type RealtimeAgentConfiguration<TContext = UnknownContext> = Partial<Omit<AgentConfiguration<TContext, TextOutput>, 'model' | 'handoffs' | 'modelSettings' | 'outputType' | 'toolUseBehavior' | 'resetToolChoice' | 'outputGuardrails' | 'inputGuardrails' | 'model'>> & {
+    /**
+     * The name of your realtime agent.
+     */
+    name: string;
+    /**
+     * Any other `RealtimeAgent` instances the agent is able to hand off to.
+     */
+    handoffs?: (RealtimeAgent | Handoff)[];
+    /**
+     * The voice intended to be used by the agent. If another agent already spoke during the
+     * RealtimeSession, changing the voice during a handoff will fail.
+     */
+    voice?: string;
+};
+/**
+ * A specialized agent instance that is meant to be used within a `RealtimeSession` to build
+ * voice agents. Due to the nature of this agent, some configuration options are not supported
+ * that are supported by regular `Agent` instances. For example:
+ * - `model` choice is not supported as all RealtimeAgents will be handled by the same model within
+ *   a `RealtimeSession`
+ * - `modelSettings` is not supported as all RealtimeAgents will be handled by the same model within
+ *   a `RealtimeSession`
+ * - `outputType` is not supported as RealtimeAgents do not support structured outputs
+ * - `toolUseBehavior` is not supported as all RealtimeAgents will be handled by the same model within
+ *   a `RealtimeSession`
+ * - `voice` can be configured on an `Agent` level however it cannot be changed after the first
+ *    agent within a `RealtimeSession` spoke
+ *
+ * @example
+ * ```ts
+ * const agent = new RealtimeAgent({
+ *   name: 'my-agent',
+ *   instructions: 'You are a helpful assistant that can answer questions and help with tasks.',
+ * })
+ *
+ * const session = new RealtimeSession(agent);
+ * ```
+ */
+export declare class RealtimeAgent<TContext = UnknownContext> extends Agent<RealtimeContextData<TContext>, TextOutput> {
+    /**
+     * The voice intended to be used by the agent. If another agent already spoke during the
+     * RealtimeSession, changing the voice during a handoff will fail.
+     */
+    readonly voice: string;
+    constructor(config: RealtimeAgentConfiguration<TContext>);
+}