@charivo/realtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +148 -0
- package/dist/chunk-BNN3F7VN.mjs +433 -0
- package/dist/chunk-BNN3F7VN.mjs.map +1 -0
- package/dist/chunk-DTNZCAFN.mjs +522 -0
- package/dist/chunk-DTNZCAFN.mjs.map +1 -0
- package/dist/chunk-XJNT4ZTF.mjs +41 -0
- package/dist/chunk-XJNT4ZTF.mjs.map +1 -0
- package/dist/index.d.mts +88 -0
- package/dist/index.d.ts +88 -0
- package/dist/index.js +874 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +864 -0
- package/dist/index.mjs.map +1 -0
- package/dist/openai/index.d.mts +54 -0
- package/dist/openai/index.d.ts +54 -0
- package/dist/openai/index.js +476 -0
- package/dist/openai/index.js.map +1 -0
- package/dist/openai/index.mjs +4 -0
- package/dist/openai/index.mjs.map +1 -0
- package/dist/openai-agents/index.d.mts +49 -0
- package/dist/openai-agents/index.d.ts +49 -0
- package/dist/openai-agents/index.js +564 -0
- package/dist/openai-agents/index.js.map +1 -0
- package/dist/openai-agents/index.mjs +4 -0
- package/dist/openai-agents/index.mjs.map +1 -0
- package/dist/remote/index.d.mts +45 -0
- package/dist/remote/index.d.ts +45 -0
- package/dist/remote/index.js +1115 -0
- package/dist/remote/index.js.map +1 -0
- package/dist/remote/index.mjs +131 -0
- package/dist/remote/index.mjs.map +1 -0
- package/dist/types-DntfJo0o.d.mts +54 -0
- package/dist/types-DntfJo0o.d.ts +54 -0
- package/package.json +64 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Zeikar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# @charivo/realtime
|
|
2
|
+
|
|
3
|
+
Provider-agnostic realtime session manager and typed config helpers for Charivo.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pnpm add @charivo/realtime
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import {
|
|
15
|
+
createRealtimeManager,
|
|
16
|
+
type RealtimeToolRegistration,
|
|
17
|
+
} from "@charivo/realtime";
|
|
18
|
+
import { createRemoteRealtimeClient } from "@charivo/realtime/remote";
|
|
19
|
+
|
|
20
|
+
const client = createRemoteRealtimeClient({ apiEndpoint: "/api/realtime" });
|
|
21
|
+
const tools: RealtimeToolRegistration[] = [
|
|
22
|
+
{
|
|
23
|
+
definition: {
|
|
24
|
+
type: "function",
|
|
25
|
+
name: "describeCharacterProfile",
|
|
26
|
+
description: "Return the active character profile.",
|
|
27
|
+
parameters: {
|
|
28
|
+
type: "object",
|
|
29
|
+
properties: {},
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
async handler(_args, context) {
|
|
33
|
+
return {
|
|
34
|
+
success: true,
|
|
35
|
+
name: context.character?.name ?? null,
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
const manager = createRealtimeManager(client, { tools });
|
|
42
|
+
manager.setCharacter({
|
|
43
|
+
id: "hiyori",
|
|
44
|
+
name: "Hiyori",
|
|
45
|
+
personality: "Cheerful and helpful assistant",
|
|
46
|
+
voice: { voiceId: "marin" },
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
await manager.startSession({
|
|
50
|
+
provider: "openai",
|
|
51
|
+
model: "gpt-realtime-mini",
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
await manager.updateSession({
|
|
55
|
+
voice: "alloy",
|
|
56
|
+
});
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Exports
|
|
60
|
+
|
|
61
|
+
- `createRealtimeManager(client, options?)`
|
|
62
|
+
- `buildRealtimeSessionConfig({ character, baseConfig? })`
|
|
63
|
+
- `createAvatarControlTools(catalog)`
|
|
64
|
+
- `DEFAULT_REALTIME_AGENT_INSTRUCTIONS`
|
|
65
|
+
- realtime-related types re-exported from `@charivo/core`
|
|
66
|
+
|
|
67
|
+
## Avatar Control Tools
|
|
68
|
+
|
|
69
|
+
Use `createAvatarControlTools(...)` to build canonical avatar tools from the
|
|
70
|
+
loaded Live2D model catalog:
|
|
71
|
+
|
|
72
|
+
- `setExpression`
|
|
73
|
+
- `playMotion`
|
|
74
|
+
- `lookAt`
|
|
75
|
+
|
|
76
|
+
## Session Refresh
|
|
77
|
+
|
|
78
|
+
`updateSession(...)` refreshes the active provider session by reconnecting with
|
|
79
|
+
the latest requested config, current character, and current tool registry.
|
|
80
|
+
|
|
81
|
+
- inactive managers only cache the requested base config for the next
|
|
82
|
+
`startSession(...)`
|
|
83
|
+
- active managers reconnect instead of attempting a transport-level patch
|
|
84
|
+
- refresh lifecycle events use `reason: "refresh"` on
|
|
85
|
+
`realtime:session:end/start`
|
|
86
|
+
- refresh failures do not roll back to the previous live session
|
|
87
|
+
- `state.session.config` is only replaced after the new connection succeeds
|
|
88
|
+
- repeated `updateSession(...)` calls are coalesced to the latest config
|
|
89
|
+
- `stopSession()` wins over an in-flight refresh and converges to a stopped
|
|
90
|
+
session
|
|
91
|
+
|
|
92
|
+
## Tool Registry
|
|
93
|
+
|
|
94
|
+
`RealtimeManager` owns the tool registry. Definitions sent to the provider come
|
|
95
|
+
from the registry, not from `defaultSessionConfig.tools`.
|
|
96
|
+
|
|
97
|
+
```ts
|
|
98
|
+
manager.registerTool({
|
|
99
|
+
definition: {
|
|
100
|
+
type: "function",
|
|
101
|
+
name: "describeScene",
|
|
102
|
+
description: "Describe the current scene.",
|
|
103
|
+
parameters: {
|
|
104
|
+
type: "object",
|
|
105
|
+
properties: {},
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
async handler() {
|
|
109
|
+
return {
|
|
110
|
+
success: true,
|
|
111
|
+
scene: "Cafe",
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Current limitation:
|
|
118
|
+
|
|
119
|
+
- `registerTool(...)` and `unregisterTool(...)` update the local manager registry immediately
|
|
120
|
+
- active provider sessions are not updated until `updateSession(...)` or the next `startSession(...)`
|
|
121
|
+
- unregistered tools may still be called by an already-active provider session and will return a failure result
|
|
122
|
+
|
|
123
|
+
## Event Bridge
|
|
124
|
+
|
|
125
|
+
`RealtimeManager` accepts an emit-only event bridge through
|
|
126
|
+
`setEventEmitter(...)`. It relays client output into the Charivo event stream,
|
|
127
|
+
but it does not subscribe through the shared event bus.
|
|
128
|
+
|
|
129
|
+
When connected, the manager relays:
|
|
130
|
+
|
|
131
|
+
- `realtime:session:start`
|
|
132
|
+
- `realtime:session:end`
|
|
133
|
+
- `realtime:state`
|
|
134
|
+
- `realtime:user:transcript`
|
|
135
|
+
- `realtime:assistant:start`
|
|
136
|
+
- `realtime:assistant:delta`
|
|
137
|
+
- `realtime:assistant:done`
|
|
138
|
+
- `realtime:tool:call`
|
|
139
|
+
- `realtime:tool:result`
|
|
140
|
+
- `realtime:tool:error`
|
|
141
|
+
- `realtime:expression`
|
|
142
|
+
- `realtime:motion`
|
|
143
|
+
- `realtime:gaze`
|
|
144
|
+
- `realtime:text:delta`
|
|
145
|
+
- `realtime:error`
|
|
146
|
+
- `tts:lipsync:update`
|
|
147
|
+
- `tts:audio:start`
|
|
148
|
+
- `tts:audio:end`
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-XJNT4ZTF.mjs';
|
|
2
|
+
import { OPENAI_REALTIME_ADAPTER } from '@charivo/core';
|
|
3
|
+
|
|
4
|
+
var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
5
|
+
"session.created",
|
|
6
|
+
"session.updated",
|
|
7
|
+
"response.created",
|
|
8
|
+
"response.done",
|
|
9
|
+
"response.function_call_arguments.done",
|
|
10
|
+
"conversation.item.input_audio_transcription.completed",
|
|
11
|
+
"error"
|
|
12
|
+
]);
|
|
13
|
+
var OpenAIRealtimeClient = class {
|
|
14
|
+
constructor(options = {}) {
|
|
15
|
+
this.options = options;
|
|
16
|
+
}
|
|
17
|
+
pc = null;
|
|
18
|
+
dc = null;
|
|
19
|
+
audioElement = null;
|
|
20
|
+
mediaStream = null;
|
|
21
|
+
audioContext = null;
|
|
22
|
+
analyser = null;
|
|
23
|
+
lipSyncInterval = null;
|
|
24
|
+
isResponseInProgress = false;
|
|
25
|
+
cancelInFlight = false;
|
|
26
|
+
hasStartedAssistantResponse = false;
|
|
27
|
+
hasStartedAudioOutput = false;
|
|
28
|
+
assistantText = "";
|
|
29
|
+
eventCallbacks = /* @__PURE__ */ new Set();
|
|
30
|
+
async connect(config) {
|
|
31
|
+
try {
|
|
32
|
+
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
33
|
+
this.pc = new RTCPeerConnection();
|
|
34
|
+
this.audioElement = document.createElement("audio");
|
|
35
|
+
this.audioElement.autoplay = true;
|
|
36
|
+
this.pc.ontrack = (event) => {
|
|
37
|
+
if (this.audioElement) {
|
|
38
|
+
this.audioElement.srcObject = event.streams[0];
|
|
39
|
+
}
|
|
40
|
+
this.setupAudioAnalysis(event.streams[0]);
|
|
41
|
+
};
|
|
42
|
+
try {
|
|
43
|
+
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
44
|
+
audio: true
|
|
45
|
+
});
|
|
46
|
+
const audioTrack = this.mediaStream.getTracks()[0];
|
|
47
|
+
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
48
|
+
} catch {
|
|
49
|
+
throw new Error("Microphone access required for Realtime API");
|
|
50
|
+
}
|
|
51
|
+
this.dc = this.pc.createDataChannel("oai-events");
|
|
52
|
+
this.dc.onmessage = (event) => {
|
|
53
|
+
const payload = JSON.parse(event.data);
|
|
54
|
+
this.handleServerEvent(payload);
|
|
55
|
+
};
|
|
56
|
+
this.dc.onerror = () => {
|
|
57
|
+
this.emitEvent({
|
|
58
|
+
type: "error",
|
|
59
|
+
error: new Error("DataChannel error")
|
|
60
|
+
});
|
|
61
|
+
};
|
|
62
|
+
const offer = await this.pc.createOffer();
|
|
63
|
+
await this.pc.setLocalDescription(offer);
|
|
64
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
65
|
+
transport: "webrtc",
|
|
66
|
+
session: config ?? {},
|
|
67
|
+
sdpOffer: offer.sdp
|
|
68
|
+
});
|
|
69
|
+
if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
await this.pc.setRemoteDescription({
|
|
75
|
+
type: "answer",
|
|
76
|
+
sdp: bootstrap.answerSdp
|
|
77
|
+
});
|
|
78
|
+
this.emitEvent({ type: "session.started" });
|
|
79
|
+
} catch (error) {
|
|
80
|
+
this.cleanup();
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async disconnect() {
|
|
85
|
+
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
86
|
+
this.cleanup();
|
|
87
|
+
}
|
|
88
|
+
async sendText(text) {
|
|
89
|
+
if (!this.dc || this.dc.readyState !== "open") {
|
|
90
|
+
throw new Error("DataChannel not ready");
|
|
91
|
+
}
|
|
92
|
+
if (this.isResponseInProgress) {
|
|
93
|
+
console.warn("\u26A0\uFE0F Response already in progress, skipping request");
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
this.dc.send(
|
|
97
|
+
JSON.stringify({
|
|
98
|
+
type: "conversation.item.create",
|
|
99
|
+
item: {
|
|
100
|
+
type: "message",
|
|
101
|
+
role: "user",
|
|
102
|
+
content: [
|
|
103
|
+
{
|
|
104
|
+
type: "input_text",
|
|
105
|
+
text
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
})
|
|
110
|
+
);
|
|
111
|
+
this.dc.send(JSON.stringify({ type: "response.create" }));
|
|
112
|
+
this.beginResponseRequest();
|
|
113
|
+
}
|
|
114
|
+
async sendAudio(_audio) {
|
|
115
|
+
console.warn(
|
|
116
|
+
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
async sendToolResult(callId, output) {
|
|
120
|
+
const dc = this.requireOpenDataChannel();
|
|
121
|
+
dc.send(
|
|
122
|
+
JSON.stringify({
|
|
123
|
+
type: "conversation.item.create",
|
|
124
|
+
item: {
|
|
125
|
+
type: "function_call_output",
|
|
126
|
+
call_id: callId,
|
|
127
|
+
output: JSON.stringify(output)
|
|
128
|
+
}
|
|
129
|
+
})
|
|
130
|
+
);
|
|
131
|
+
dc.send(JSON.stringify({ type: "response.create" }));
|
|
132
|
+
this.beginResponseRequest();
|
|
133
|
+
}
|
|
134
|
+
async interrupt() {
|
|
135
|
+
const dc = this.requireOpenDataChannel();
|
|
136
|
+
if (!this.isResponseInProgress) {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
dc.send(JSON.stringify({ type: "response.cancel" }));
|
|
140
|
+
this.isResponseInProgress = false;
|
|
141
|
+
this.cancelInFlight = true;
|
|
142
|
+
}
|
|
143
|
+
onEvent(callback) {
|
|
144
|
+
this.eventCallbacks.add(callback);
|
|
145
|
+
}
|
|
146
|
+
handleServerEvent(event) {
|
|
147
|
+
if (this.options.debug && this.shouldLogDebugEvent(event.type)) {
|
|
148
|
+
this.log("\u{1F4E1} [OpenAI Realtime Event]", event.type, event);
|
|
149
|
+
}
|
|
150
|
+
switch (event.type) {
|
|
151
|
+
case "session.created":
|
|
152
|
+
case "session.updated":
|
|
153
|
+
return;
|
|
154
|
+
case "response.audio.delta":
|
|
155
|
+
case "output_audio_buffer.started":
|
|
156
|
+
if (this.cancelInFlight) {
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
if (!this.hasStartedAudioOutput) {
|
|
160
|
+
this.hasStartedAudioOutput = true;
|
|
161
|
+
this.emitEvent({ type: "audio.output.started" });
|
|
162
|
+
}
|
|
163
|
+
return;
|
|
164
|
+
case "response.audio.done":
|
|
165
|
+
case "response.output_audio.done":
|
|
166
|
+
case "output_audio_buffer.stopped":
|
|
167
|
+
if (this.cancelInFlight) {
|
|
168
|
+
this.hasStartedAudioOutput = false;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
this.emitEvent({ type: "audio.output.ended" });
|
|
172
|
+
this.hasStartedAudioOutput = false;
|
|
173
|
+
return;
|
|
174
|
+
case "response.done":
|
|
175
|
+
this.isResponseInProgress = false;
|
|
176
|
+
if (this.cancelInFlight) {
|
|
177
|
+
this.cancelInFlight = false;
|
|
178
|
+
this.resetResponseTracking();
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
this.emitEvent({
|
|
182
|
+
type: "assistant.response.completed",
|
|
183
|
+
text: this.assistantText
|
|
184
|
+
});
|
|
185
|
+
this.resetResponseTracking();
|
|
186
|
+
return;
|
|
187
|
+
case "response.audio_transcript.delta":
|
|
188
|
+
case "response.output_audio_transcript.delta":
|
|
189
|
+
case "response.output_text.delta":
|
|
190
|
+
if (this.cancelInFlight || !event.delta) {
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
if (!this.hasStartedAssistantResponse) {
|
|
194
|
+
this.hasStartedAssistantResponse = true;
|
|
195
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
196
|
+
}
|
|
197
|
+
this.assistantText += event.delta;
|
|
198
|
+
this.emitEvent({
|
|
199
|
+
type: "assistant.text.delta",
|
|
200
|
+
text: event.delta
|
|
201
|
+
});
|
|
202
|
+
return;
|
|
203
|
+
case "response.output_text.done":
|
|
204
|
+
if (this.cancelInFlight || !event.text) {
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
if (!this.hasStartedAssistantResponse) {
|
|
208
|
+
this.hasStartedAssistantResponse = true;
|
|
209
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
210
|
+
}
|
|
211
|
+
if (event.text !== this.assistantText) {
|
|
212
|
+
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
213
|
+
this.assistantText = event.text;
|
|
214
|
+
if (delta) {
|
|
215
|
+
this.emitEvent({
|
|
216
|
+
type: "assistant.text.delta",
|
|
217
|
+
text: delta
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return;
|
|
222
|
+
case "response.output_audio_transcript.done":
|
|
223
|
+
if (this.cancelInFlight || !event.transcript) {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (!this.hasStartedAssistantResponse) {
|
|
227
|
+
this.hasStartedAssistantResponse = true;
|
|
228
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
229
|
+
}
|
|
230
|
+
if (event.transcript !== this.assistantText) {
|
|
231
|
+
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
232
|
+
this.assistantText = event.transcript;
|
|
233
|
+
if (delta) {
|
|
234
|
+
this.emitEvent({
|
|
235
|
+
type: "assistant.text.delta",
|
|
236
|
+
text: delta
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return;
|
|
241
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
242
|
+
if (event.transcript) {
|
|
243
|
+
this.emitEvent({
|
|
244
|
+
type: "user.transcript",
|
|
245
|
+
text: event.transcript
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
return;
|
|
249
|
+
case "response.function_call_arguments.done":
|
|
250
|
+
this.handleToolCallDone(event);
|
|
251
|
+
return;
|
|
252
|
+
case "error":
|
|
253
|
+
this.isResponseInProgress = false;
|
|
254
|
+
this.cancelInFlight = false;
|
|
255
|
+
this.emitEvent({
|
|
256
|
+
type: "error",
|
|
257
|
+
error: new Error(event.error?.message || "Unknown error")
|
|
258
|
+
});
|
|
259
|
+
return;
|
|
260
|
+
default:
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
handleToolCallDone(event) {
|
|
265
|
+
const callId = event.call_id || event.item?.call_id || event.item_id;
|
|
266
|
+
const name = event.name || event.item?.name;
|
|
267
|
+
const argsJson = event.arguments || event.item?.arguments;
|
|
268
|
+
if (!name || !argsJson) {
|
|
269
|
+
console.warn("\u26A0\uFE0F Tool call done but missing name or arguments");
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
let args = {};
|
|
273
|
+
try {
|
|
274
|
+
const parsed = JSON.parse(argsJson);
|
|
275
|
+
if (isRecord(parsed)) {
|
|
276
|
+
args = parsed;
|
|
277
|
+
}
|
|
278
|
+
} catch (error) {
|
|
279
|
+
console.error("Failed to parse tool call args", error, argsJson);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
this.emitEvent({
|
|
283
|
+
type: "tool.call",
|
|
284
|
+
name,
|
|
285
|
+
args,
|
|
286
|
+
callId
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
async getSessionBootstrap(request) {
|
|
290
|
+
if (this.options.sessionBootstrap) {
|
|
291
|
+
return this.options.sessionBootstrap(request);
|
|
292
|
+
}
|
|
293
|
+
const apiEndpoint = this.options.apiEndpoint;
|
|
294
|
+
if (!apiEndpoint) {
|
|
295
|
+
throw new Error(
|
|
296
|
+
"OpenAI realtime client requires apiEndpoint or sessionBootstrap"
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
const response = await fetchWithTimeout(
|
|
300
|
+
apiEndpoint,
|
|
301
|
+
{
|
|
302
|
+
method: "POST",
|
|
303
|
+
headers: {
|
|
304
|
+
"Content-Type": "application/json"
|
|
305
|
+
},
|
|
306
|
+
body: JSON.stringify(request)
|
|
307
|
+
},
|
|
308
|
+
`Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
|
|
309
|
+
);
|
|
310
|
+
if (!response.ok) {
|
|
311
|
+
const errorText = await response.text();
|
|
312
|
+
throw new Error(`Failed to create Realtime session: ${errorText}`);
|
|
313
|
+
}
|
|
314
|
+
const bootstrap = await response.json();
|
|
315
|
+
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
316
|
+
throw new Error("Invalid realtime session bootstrap response");
|
|
317
|
+
}
|
|
318
|
+
return bootstrap;
|
|
319
|
+
}
|
|
320
|
+
setupAudioAnalysis(stream) {
|
|
321
|
+
try {
|
|
322
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
323
|
+
if (!audioContextConstructor) {
|
|
324
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
325
|
+
}
|
|
326
|
+
this.audioContext = new audioContextConstructor();
|
|
327
|
+
const source = this.audioContext.createMediaStreamSource(stream);
|
|
328
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
329
|
+
this.analyser.fftSize = 256;
|
|
330
|
+
this.analyser.smoothingTimeConstant = 0.8;
|
|
331
|
+
source.connect(this.analyser);
|
|
332
|
+
this.startLipSyncAnalysis();
|
|
333
|
+
} catch (error) {
|
|
334
|
+
console.error("Failed to setup audio analysis:", error);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
startLipSyncAnalysis() {
|
|
338
|
+
if (!this.analyser) {
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
342
|
+
const dataArray = new Uint8Array(bufferLength);
|
|
343
|
+
this.lipSyncInterval = window.setInterval(() => {
|
|
344
|
+
if (!this.analyser) {
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
this.analyser.getByteFrequencyData(dataArray);
|
|
348
|
+
let sum = 0;
|
|
349
|
+
for (let index = 0; index < bufferLength; index += 1) {
|
|
350
|
+
const normalized = dataArray[index] / 255;
|
|
351
|
+
sum += normalized * normalized;
|
|
352
|
+
}
|
|
353
|
+
const rms = Math.sqrt(sum / bufferLength);
|
|
354
|
+
this.emitEvent({
|
|
355
|
+
type: "audio.lipsync",
|
|
356
|
+
rms: Math.min(rms * 3, 1)
|
|
357
|
+
});
|
|
358
|
+
}, 1e3 / 60);
|
|
359
|
+
}
|
|
360
|
+
stopLipSyncAnalysis() {
|
|
361
|
+
if (this.lipSyncInterval) {
|
|
362
|
+
clearInterval(this.lipSyncInterval);
|
|
363
|
+
this.lipSyncInterval = null;
|
|
364
|
+
}
|
|
365
|
+
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
366
|
+
}
|
|
367
|
+
cleanup() {
|
|
368
|
+
this.stopLipSyncAnalysis();
|
|
369
|
+
if (this.dc) {
|
|
370
|
+
this.dc.close();
|
|
371
|
+
this.dc = null;
|
|
372
|
+
}
|
|
373
|
+
if (this.pc) {
|
|
374
|
+
this.pc.close();
|
|
375
|
+
this.pc = null;
|
|
376
|
+
}
|
|
377
|
+
if (this.mediaStream) {
|
|
378
|
+
this.mediaStream.getTracks().forEach((track) => track.stop());
|
|
379
|
+
this.mediaStream = null;
|
|
380
|
+
}
|
|
381
|
+
if (this.audioElement) {
|
|
382
|
+
this.audioElement.srcObject = null;
|
|
383
|
+
this.audioElement = null;
|
|
384
|
+
}
|
|
385
|
+
if (this.audioContext) {
|
|
386
|
+
this.audioContext.close();
|
|
387
|
+
this.audioContext = null;
|
|
388
|
+
}
|
|
389
|
+
this.analyser = null;
|
|
390
|
+
this.isResponseInProgress = false;
|
|
391
|
+
this.cancelInFlight = false;
|
|
392
|
+
this.resetResponseTracking();
|
|
393
|
+
}
|
|
394
|
+
beginResponseRequest() {
|
|
395
|
+
this.resetResponseTracking();
|
|
396
|
+
this.isResponseInProgress = true;
|
|
397
|
+
this.cancelInFlight = false;
|
|
398
|
+
}
|
|
399
|
+
requireOpenDataChannel() {
|
|
400
|
+
if (!this.dc || this.dc.readyState !== "open") {
|
|
401
|
+
throw new Error("DataChannel not ready");
|
|
402
|
+
}
|
|
403
|
+
return this.dc;
|
|
404
|
+
}
|
|
405
|
+
resetResponseTracking() {
|
|
406
|
+
this.assistantText = "";
|
|
407
|
+
this.hasStartedAssistantResponse = false;
|
|
408
|
+
this.hasStartedAudioOutput = false;
|
|
409
|
+
}
|
|
410
|
+
emitEvent(event) {
|
|
411
|
+
for (const callback of this.eventCallbacks) {
|
|
412
|
+
callback(event);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
log(...args) {
|
|
416
|
+
if (this.options.debug) {
|
|
417
|
+
console.log(...args);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
shouldLogDebugEvent(eventType) {
|
|
421
|
+
if (DEBUG_EVENT_ALLOWLIST.has(eventType)) {
|
|
422
|
+
return true;
|
|
423
|
+
}
|
|
424
|
+
return eventType.startsWith("response.output_text.");
|
|
425
|
+
}
|
|
426
|
+
};
|
|
427
|
+
function createOpenAIRealtimeClient(options) {
|
|
428
|
+
return new OpenAIRealtimeClient(options);
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
export { OpenAIRealtimeClient, createOpenAIRealtimeClient };
|
|
432
|
+
//# sourceMappingURL=chunk-BNN3F7VN.mjs.map
|
|
433
|
+
//# sourceMappingURL=chunk-BNN3F7VN.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/openai/client.ts"],"names":[],"mappings":";;;AA6CA,IAAM,qBAAA,uBAA4B,GAAA,CAAY;AAAA,EAC5C,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,kBAAA;AAAA,EACA,eAAA;AAAA,EACA,uCAAA;AAAA,EACA,uDAAA;AAAA,EACA;AACF,CAAC,CAAA;AAQM,IAAM,uBAAN,MAA8D;AAAA,EAenE,WAAA,CAAoB,OAAA,GAAuC,EAAC,EAAG;AAA3C,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAA4C;AAAA,EAdxD,EAAA,GAA+B,IAAA;AAAA,EAC/B,EAAA,GAA4B,IAAA;AAAA,EAC5B,YAAA,GAAwC,IAAA;AAAA,EACxC,WAAA,GAAkC,IAAA;AAAA,EAClC,YAAA,GAAoC,IAAA;AAAA,EACpC,QAAA,GAAgC,IAAA;AAAA,EAChC,eAAA,GAAiC,IAAA;AAAA,EACjC,oBAAA,GAAuB,KAAA;AAAA,EACvB,cAAA,GAAiB,KAAA;AAAA,EACjB,2BAAA,GAA8B,KAAA;AAAA,EAC9B,qBAAA,GAAwB,KAAA;AAAA,EACxB,aAAA,GAAgB,EAAA;AAAA,EAChB,cAAA,uBAAqB,GAAA,EAA6C;AAAA,EAI1E,MAAM,QAAQ,MAAA,EAA+C;AAC3D,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,IAAI,4CAA4C,CAAA;AAErD,MAAA,IAAA,CAAK,EAAA,GAAK,IAAI,iBAAA,EAAkB;AAChC,MAAA,IAAA,CAAK,YAAA,GAAe,QAAA,CAAS,aAAA,CAAc,OAAO,CAAA;AAClD,MAAA,IAAA,CAAK,aAAa,QAAA,GAAW,IAAA;AAE7B,MAAA,IAAA,CAAK,EAAA,CAAG,OAAA,GAAU,CAAC,KAAA,KAAU;AAC3B,QAAA,IAAI,KAAK,YAAA,EAAc;AACrB,UAAA,IAAA,CAAK,YAAA,CAAa,SAAA,GAAY,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAA;AAAA,QAC/C;AACA,QAAA,IAAA,CAAK,kBAAA,CAAmB,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA;AAAA,MAC1C,CAAA;AAEA,MAAA,IAAI;AACF,QAAA,IAAA,CAAK,WAAA,GAAc,MAAM,SAAA,CAAU,YAAA,CAAa,YAAA,CAAa;AAAA,UAC3D,KAAA,EAAO;AAAA,SACR,CAAA;AACD,QAAA,MAAM,UAAA,GAAa,IAAA,CAAK,WAAA,CAAY,SAAA,GAAY,CAAC,CAAA;AACjD,QAAA,IAAA,CAAK,EAAA,CAAG,QAAA,CAAS,UAAA,EAAY,IAAA,CAAK,WAAW,CAAA;AAAA,MAC/C,CAAA,CAAA,MAAQ;AACN,QAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,MAC/D;AAEA,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA,CAAK,EAAA,CAAG,iBAAA,CAAkB,YAAY,CAAA;AAChD,MAAA,IAAA,CAAK,EAAA,CAAG,SAAA,GAAY,CAAC,KAAA,KAAU;AAC7B,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,KAAA,CAAM,KAAA,CAAM,IAAI,CAAA;AACrC,QAAA,IAAA,CAAK,kBAAkB,OAAO,CAAA;AAAA,MAChC,CAAA;AACA,MAAA,IAAA,CAAK,EAAA,CAAG,UAAU,MAAM;AACtB,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,OAAA;AAAA,UACN,KAAA,EAAO,IAAI,KAAA,CAAM,mBAAmB;AAAA,SACrC,CAAA;AAAA,MACH,CAAA;AAEA,MAAA,MAAM,KAAA,GAAQ,MAAM,IAAA,CAAK,EAAA,CAAG,WAAA,EAAY;AACxC,MAAA,MAAM,IAAA,CAAK,EAAA,CAAG,mBAAA,CAAoB,KAAK,CAAA;AAEvC,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,mBAAA,CAAoB;AAAA,QAC/C,SAAA,EAAW,QAAA;AAAA,QACX,OAAA,EAAS,UAAU,EAAC;AAAA,QACpB,UAAU,KAAA,CAAM;AAAA,OACjB,CAAA;AAED,MAAA,IACE,SAAA,CAAU,YAAY,uBAAA,IACtB,SAAA,CAAU,cAAc,QAAA,IACxB,EAAE,eAAe,SAAA,CAAA,EACjB;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,wCAAwC,uBAAuB,CAAA,qBAAA,EAAwB,UAAU,OAAO,CAAA,CAAA,EAAI,UAAU,SAAS,CAAA;AAAA,SACjI;AAAA,MACF;AAEA,MAAA,MAAM,IAAA,CAAK,GAAG,oBAAA,CAAqB;AAAA,QACjC,IAAA,EAAM,QAAA;AAAA,QACN,KAAK,SAAA,CAAU;AAAA,OAChB,CAAA;AAED,MAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAA;AAAA,IAC5C,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,EAAQ;AACb,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,IAAI,sCAAsC,CAAA;AAC/C,IAAA,IAAA,CAAK,OAAA,EAAQ;AAAA,EACf;AAAA,EAEA,MAAM,SAAS,IAAA,EAA6B;AAC1C,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,OAAA,CAAQ,KAAK,6DAAmD,CAAA;AAChE,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,EAAA,CAAG,IAAA;AAAA,MACN,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,SAAA;AAAA,UACN,IAAA,EAAM,MAAA;AAAA,UACN,OAAA,EAAS;AAAA,YACP;AAAA,cACE,IAAA,EAAM,YAAA;AAAA,cACN;AAAA;AACF;AACF;AACF,OACD;AAAA,KACH;AACA,IAAA,IAAA,CAAK,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAExD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,UAAU,MAAA,EAAoC;AAClD,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN;AAAA,KACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAA,CACJ,MAAA,EACA,MAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,EAAA,CAAG,IAAA;AAAA,MACD,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,sBAAA;AAAA,UACN,OAAA,EAAS,MAAA;AAAA,UACT,MAAA,EAAQ,IAAA,CAAK,SAAA,CAAU,MAAM;AAAA;AAC/B,OACD;AAAA,KACH;AACA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAEnD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,SAAA,GAA2B;AAC/B,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AACnD,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,IAAA;AAAA,EACxB;AAAA,EAEA,QAAQ,QAAA,EAAyD;AAC/D,IAAA,IAAA,CAAK,cAAA,CAAe,IAAI,QAAQ,CAAA;AAAA,EAClC;AAAA,EAEQ,kBAAkB,KAAA,EAA0B;AAClD,IAAA,IAAI,KAAK,OAAA,CAAQ,KAAA,IAAS,KAAK,mBAAA,CAAoB,KAAA,CAAM,IAAI,CAAA,EAAG;AAC9D,MAAA,IAAA,CAAK,GAAA,CAAI,mCAAA,EAA8B,KAAA,CAAM,IAAA,EAAM,KAAK,CAAA;AAAA,IAC1D;AAEA,IAAA,QAAQ,MAAM,IAAA;AAAM,MAClB,KAAK,iBAAA;AAAA,MACL,KAAK,iBAAA;AACH,QAAA;AAAA,MAEF,KAAK,sBAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,qBAAA,EAAuB;AAC/B,UAAA,IAAA,CAAK,qBAAA,GAAwB,IAAA;AAC7B,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,sBAAA,EAAwB,CAAA;AAAA,QACjD;AACA,QAAA;AAAA,MAEF,KAAK,qBAAA;AAAA,MACL,KAAK,4BAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,oBAAA,EAAsB,CAAA;AAC7C,QAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,eAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,UAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,UAAA;AAAA,QACF;AACA,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,8BAAA;AAAA,UACN,MAAM,IAAA,CAAK;AAAA,SACZ,CAAA;AACD,QAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,QAAA;AAAA,MAEF,KAAK,iCAAA;AAAA,MACL,KAAK,wCAAA;AAAA,MACL,KAAK,4BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,KAAA,EAAO;AACvC,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAA,CAAK,iBAAiB,KAAA,CAAM,KAAA;AAC5B,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,sBAAA;AAAA,UACN,MAAM,KAAA,CAAM;AAAA,SACb,CAAA;AACD,QAAA;AAAA,MAEF,KAAK,2BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,IAAA,EAAM;AACtC,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAI,KAAA,CAAM,IAAA,KAAS,IAAA,CAAK,aAAA,EAAe;AACrC,UAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,IAAA,CAAK,UAAA,CAAW,KAAK,aAAa,CAAA,GAClD,KAAA,CAAM,IAAA,CAAK,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,MAAM,IAC1C,KAAA,CAAM,IAAA;AAEV,UAAA,IAAA,CAAK,gBAAgB,KAAA,CAAM,IAAA;AAC3B,UAAA,IAAI,KAAA,EAAO;AACT,YAAA,IAAA,CAAK,SAAA,CAAU;AAAA,cACb,IAAA,EAAM,sBAAA;AAAA,cACN,IAAA,EAAM;AAAA,aACP,CAAA;AAAA,UACH;AAAA,QACF;AACA,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,UAAA,EAAY;AAC5C,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAI,KAAA,CAAM,UAAA,KAAe,IAAA,CAAK,aAAA,EAAe;AAC3C,UAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,UAAA,CAAW,UAAA,CAAW,KAAK,aAAa,CAAA,GACxD,KAAA,CAAM,UAAA,CAAW,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,MAAM,IAChD,KAAA,CAAM,UAAA;AAEV,UAAA,IAAA,CAAK,gBAAgB,KAAA,CAAM,UAAA;AAC3B,UAAA,IAAI,KAAA,EAAO;AACT,YAAA,IAAA,CAAK,SAAA,CAAU;AAAA,cACb,IAAA,EAAM,sBAAA;AAAA,cACN,IAAA,EAAM;AAAA,aACP,CAAA;AAAA,UACH;AAAA,QACF;AACA,QAAA;AAAA,MAEF,KAAK,uDAAA;AACH,QAAA,IAAI,MAAM,UAAA,EAAY;AACpB,UAAA,IAAA,CAAK,SAAA,CAAU;AAAA,YACb,IAAA,EAAM,iBAAA;AAAA,YACN,MAAM,KAAA,CAAM;AAAA,WACb,CAAA;AAAA,QACH;AACA,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAA,CAAK,mBAAmB,KAAK,CAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,OAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,OAAA;AAAA,UACN,OAAO,IAAI,KAAA,CAAM,KAAA,CAAM,KAAA,EAAO,WAAW,eAAe;AAAA,SACzD,CAAA;AACD,QAAA;AAAA,MAEF;AACE,QAAA;AAAA;AACJ,EACF;AAAA,EAEQ,mBAAmB,KAAA,EAA0B;AACnD,IAAA,MAAM,SAAS,KAAA,CAAM,OAAA,IAAW,KAAA,CAAM,IAAA,EAAM,WAAW,KAAA,CAAM,OAAA;AAC7D,IAAA,MAAM,IAAA,GAAO,KAAA,CAAM,IAAA,IAAQ,KAAA,CAAM,IAAA,EAAM,IAAA;AACvC,IAAA,MAAM,QAAA,GAAW,KAAA,CAAM,SAAA,IAAa,KAAA,CAAM,IAAA,EAAM,SAAA;AAEhD,IAAA,IAAI,CAAC,IAAA,IAAQ,CAAC,QAAA,EAAU;AACtB,MAAA,OAAA,CAAQ,KAAK,2DAAiD,CAAA;AAC9D,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,OAAgC,EAAC;AACrC,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,QAAQ,CAAA;AAClC,MAAA,IAAI,QAAA,CAAS,MAAM,CAAA,EAAG;AACpB,QAAA,IAAA,GAAO,MAAA;AAAA,MACT;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,gCAAA,EAAkC,KAAA,EAAO,QAAQ,CAAA;AAC/D,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,SAAA,CAAU;AAAA,MACb,IAAA,EAAM,WAAA;AAAA,MACN,IAAA;AAAA,MACA,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,oBACZ,OAAA,EACmC;AACnC,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,OAAO,IAAA,CAAK,OAAA,CAAQ,gBAAA,CAAiB,OAAO,CAAA;AAAA,IAC9C;AAEA,IAAA,MAAM,WAAA,GAAc,KAAK,OAAA,CAAQ,WAAA;AACjC,IAAA,IAAI,CAAC,WAAA,EAAa;AAChB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,MAAM,WAAW,MAAM,gBAAA;AAAA,MACrB,WAAA;AAAA,MACA;AAAA,QACE,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB;AAAA,SAClB;AAAA,QACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,OAAO;AAAA,OAC9B;AAAA,MACA,4CAA4C,0BAA0B,CAAA,EAAA;AAAA,KACxE;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mCAAA,EAAsC,SAAS,CAAA,CAAE,CAAA;AAAA,IACnE;AAEA,IAAA,MAAM,SAAA,GAAa,MAAM,QAAA,CAAS,IAAA,EAAK;AACvC,IAAA,IAAI,CAAC,0BAAA,CAA2B,SAAS,CAAA,EAAG;AAC1C,MAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,IAC/D;AAEA,IAAA,OAAO,SAAA;AAAA,EACT;AAAA,EAEQ,mBAAmB,MAAA,EAA2B;AACpD,IAAA,IAAI;AACF,MAAA,MAAM,uBAAA,GACJ,MAAA,CAAO,YAAA,IACN,MAAA,CACE,kBAAA;AACL,MAAA,IAAI,CAAC,uBAAA,EAAyB;AAC5B,QAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AAAA,MACjE;AAEA,MAAA,IAAA,CAAK,YAAA,GAAe,IAAI,uBAAA,EAAwB;AAChD,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,YAAA,CAAa,uBAAA,CAAwB,MAAM,CAAA;AAC/D,MAAA,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,YAAA,CAAa,cAAA,EAAe;AACjD,MAAA,IAAA,CAAK,SAAS,OAAA,GAAU,GAAA;AACxB,MAAA,IAAA,CAAK,SAAS,qBAAA,GAAwB,GAAA;AACtC,MAAA,MAAA,CAAO,OAAA,CAAQ,KAAK,QAAQ,CAAA;AAE5B,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,IAC5B,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,mCAAmC,KAAK,CAAA;AAAA,IACxD;AAAA,EACF;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,QAAA,CAAS,iBAAA;AACnC,IAAA,MAAM,SAAA,GAAY,IAAI,UAAA,CAAW,YAAY,CAAA;AAE7C,IAAA,IAAA,CAAK,eAAA,GAAkB,MAAA,CAAO,WAAA,CAAY,MAAM;AAC9C,MAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,QAAA,CAAS,qBAAqB,SAAS,CAAA;AAE5C,MAAA,IAAI,GAAA,GAAM,CAAA;AACV,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,YAAA,EAAc,SAAS,CAAA,EAAG;AACpD,QAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAK,CAAA,GAAI,GAAA;AACtC,QAAA,GAAA,IAAO,UAAA,GAAa,UAAA;AAAA,MACtB;AAEA,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,IAAA,CAAK,GAAA,GAAM,YAAY,CAAA;AACxC,MAAA,IAAA,CAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,eAAA;AAAA,QACN,GAAA,EAAK,IAAA,CAAK,GAAA,CAAI,GAAA,GAAM,GAAG,CAAC;AAAA,OACzB,CAAA;AAAA,IACH,CAAA,EAAG,MAAO,EAAE,CAAA;AAAA,EACd;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,aAAA,CAAc,KAAK,eAAe,CAAA;AAClC,MAAA,IAAA,CAAK,eAAA,GAAkB,IAAA;AAAA,IACzB;AAEA,IAAA,IAAA,CAAK,UAAU,EAAE,IAAA,EAAM,eAAA,EAAiB,GAAA,EAAK,GAAG,CAAA;AAAA,EAClD;AAAA,EAEQ,OAAA,GAAgB;AACtB,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAEzB,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,WAAA,EAAa;AACpB,MAAA,IAAA,CAAK,WAAA,CAAY,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAC5D,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAAA,IACrB;AAEA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAA,CAAK,aAAa,SAAA,GAAY,IAAA;AAC9B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAA,CAAK,aAAa,KAAA,EAAM;AACxB,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,QAAA,GAAW,IAAA;AAChB,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAAA,EAC7B;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AAAA,EACxB;AAAA,EAEQ,sBAAA,GAAyC;AAC/C,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,OAAO,IAAA,CAAK,EAAA;AAAA,EACd;AAAA,EAEQ,qBAAA,GAA8B;AACpC,IAAA,IAAA,CAAK,aAAA,GAAgB,EAAA;AACrB,IAAA,IAAA,CAAK,2BAAA,GAA8B,KAAA;AACnC,IAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAAA,EAC/B;AAAA,EAEQ,UAAU,KAAA,EAAqC;AACrD,IAAA,KAAA,MAAW,QAAA,IAAY,KAAK,cAAA,EAAgB;AAC1C,MAAA,QAAA,CAAS,KAAK,CAAA;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,OAAO,IAAA,EAAuB;AACpC,IAAA,IAAI,IAAA,CAAK,QAAQ,KAAA,EAAO;AACtB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAG,IAAI,CAAA;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,oBAAoB,SAAA,EAA4B;AACtD,IAAA,IAAI,qBAAA,CAAsB,GAAA,CAAI,SAAS,CAAA,EAAG;AACxC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,SAAA,CAAU,WAAW,uBAAuB,CAAA;AAAA,EACrD;AACF;AAEO,SAAS,2BACd,OAAA,EACyB;AACzB,EAAA,OAAO,IAAI,qBAAqB,OAAO,CAAA;AACzC","file":"chunk-BNN3F7VN.mjs","sourcesContent":["import type {\n RealtimeSessionBootstrap,\n RealtimeSessionConfig,\n RealtimeSessionRequest,\n} from \"@charivo/core\";\nimport { OPENAI_REALTIME_ADAPTER } from \"@charivo/core\";\nimport {\n DEFAULT_REQUEST_TIMEOUT_MS,\n fetchWithTimeout,\n isRealtimeSessionBootstrap,\n isRecord,\n} from \"../internal/shared\";\nimport type { RealtimeTransportClient, RealtimeTransportEvent } from \"../types\";\n\ninterface ServerError {\n message?: string;\n}\n\ninterface ServerEventItem {\n call_id?: string;\n name?: string;\n arguments?: string;\n}\n\ninterface ServerEvent {\n type: string;\n delta?: string;\n text?: string;\n transcript?: string;\n call_id?: string;\n item_id?: string;\n name?: string;\n arguments?: string;\n item?: ServerEventItem;\n error?: ServerError;\n}\n\nexport interface OpenAIRealtimeClientOptions {\n apiEndpoint?: string;\n debug?: boolean;\n sessionBootstrap?: (\n request: RealtimeSessionRequest,\n ) => Promise<RealtimeSessionBootstrap>;\n}\n\nconst DEBUG_EVENT_ALLOWLIST = new Set<string>([\n \"session.created\",\n \"session.updated\",\n \"response.created\",\n \"response.done\",\n \"response.function_call_arguments.done\",\n \"conversation.item.input_audio_transcription.completed\",\n \"error\",\n]);\n\n/**\n * OpenAI-specific realtime transport client.\n *\n * This package normalizes OpenAI Realtime WebRTC events into the transport\n * event contract defined by `@charivo/realtime`.\n */\nexport class OpenAIRealtimeClient implements RealtimeTransportClient {\n private pc: RTCPeerConnection | null = null;\n private dc: RTCDataChannel | null = null;\n private audioElement: HTMLAudioElement | null = null;\n private mediaStream: MediaStream | null = null;\n private audioContext: AudioContext | null = null;\n private analyser: AnalyserNode | null = null;\n private lipSyncInterval: number | null = null;\n private isResponseInProgress = false;\n private cancelInFlight = false;\n private hasStartedAssistantResponse = false;\n private hasStartedAudioOutput = false;\n private assistantText = \"\";\n private eventCallbacks = new Set<(event: RealtimeTransportEvent) => void>();\n\n constructor(private options: OpenAIRealtimeClientOptions = {}) {}\n\n async connect(config?: RealtimeSessionConfig): Promise<void> {\n try {\n this.log(\"Starting OpenAI Realtime WebRTC connection\");\n\n this.pc = new RTCPeerConnection();\n this.audioElement = document.createElement(\"audio\");\n this.audioElement.autoplay = true;\n\n this.pc.ontrack = (event) => {\n if (this.audioElement) {\n this.audioElement.srcObject = event.streams[0];\n }\n this.setupAudioAnalysis(event.streams[0]);\n };\n\n try {\n this.mediaStream = await navigator.mediaDevices.getUserMedia({\n audio: true,\n });\n const audioTrack = this.mediaStream.getTracks()[0];\n this.pc.addTrack(audioTrack, this.mediaStream);\n } catch {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n this.dc = this.pc.createDataChannel(\"oai-events\");\n this.dc.onmessage = (event) => {\n const payload = JSON.parse(event.data) as ServerEvent;\n this.handleServerEvent(payload);\n };\n this.dc.onerror = () => {\n this.emitEvent({\n type: \"error\",\n error: new Error(\"DataChannel error\"),\n });\n };\n\n const offer = await this.pc.createOffer();\n await this.pc.setLocalDescription(offer);\n\n const bootstrap = await this.getSessionBootstrap({\n transport: \"webrtc\",\n session: config ?? {},\n sdpOffer: offer.sdp,\n });\n\n if (\n bootstrap.adapter !== OPENAI_REALTIME_ADAPTER ||\n bootstrap.transport !== \"webrtc\" ||\n !(\"answerSdp\" in bootstrap)\n ) {\n throw new Error(\n `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`,\n );\n }\n\n await this.pc.setRemoteDescription({\n type: \"answer\",\n sdp: bootstrap.answerSdp,\n });\n\n this.emitEvent({ type: \"session.started\" });\n } catch (error) {\n this.cleanup();\n throw error;\n }\n }\n\n async disconnect(): Promise<void> {\n this.log(\"Disconnecting OpenAI Realtime WebRTC\");\n this.cleanup();\n }\n\n async sendText(text: string): Promise<void> {\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n if (this.isResponseInProgress) {\n console.warn(\"⚠️ Response already in progress, skipping request\");\n return;\n }\n\n this.dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text,\n },\n ],\n },\n }),\n );\n this.dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async sendAudio(_audio: ArrayBuffer): Promise<void> {\n console.warn(\n \"sendAudio is not needed with WebRTC - audio is automatically transmitted\",\n );\n }\n\n async sendToolResult(\n callId: string,\n output: Record<string, unknown>,\n ): Promise<void> {\n const dc = this.requireOpenDataChannel();\n\n dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"function_call_output\",\n call_id: callId,\n output: JSON.stringify(output),\n },\n }),\n );\n dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async interrupt(): Promise<void> {\n const dc = this.requireOpenDataChannel();\n\n if (!this.isResponseInProgress) {\n return;\n }\n\n dc.send(JSON.stringify({ type: \"response.cancel\" }));\n this.isResponseInProgress = false;\n this.cancelInFlight = true;\n }\n\n onEvent(callback: (event: RealtimeTransportEvent) => void): void {\n this.eventCallbacks.add(callback);\n }\n\n private handleServerEvent(event: ServerEvent): void {\n if (this.options.debug && this.shouldLogDebugEvent(event.type)) {\n this.log(\"📡 [OpenAI Realtime Event]\", event.type, event);\n }\n\n switch (event.type) {\n case \"session.created\":\n case \"session.updated\":\n return;\n\n case \"response.audio.delta\":\n case \"output_audio_buffer.started\":\n if (this.cancelInFlight) {\n return;\n }\n\n if (!this.hasStartedAudioOutput) {\n this.hasStartedAudioOutput = true;\n this.emitEvent({ type: \"audio.output.started\" });\n }\n return;\n\n case \"response.audio.done\":\n case \"response.output_audio.done\":\n case \"output_audio_buffer.stopped\":\n if (this.cancelInFlight) {\n this.hasStartedAudioOutput = false;\n return;\n }\n\n this.emitEvent({ type: \"audio.output.ended\" });\n this.hasStartedAudioOutput = false;\n return;\n\n case \"response.done\":\n this.isResponseInProgress = false;\n if (this.cancelInFlight) {\n this.cancelInFlight = false;\n this.resetResponseTracking();\n return;\n }\n this.emitEvent({\n type: \"assistant.response.completed\",\n text: this.assistantText,\n });\n this.resetResponseTracking();\n return;\n\n case \"response.audio_transcript.delta\":\n case \"response.output_audio_transcript.delta\":\n case \"response.output_text.delta\":\n if (this.cancelInFlight || !event.delta) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n this.assistantText += event.delta;\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: event.delta,\n });\n return;\n\n case \"response.output_text.done\":\n if (this.cancelInFlight || !event.text) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n if (event.text !== this.assistantText) {\n const delta = event.text.startsWith(this.assistantText)\n ? event.text.slice(this.assistantText.length)\n : event.text;\n\n this.assistantText = event.text;\n if (delta) {\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: delta,\n });\n }\n }\n return;\n\n case \"response.output_audio_transcript.done\":\n if (this.cancelInFlight || !event.transcript) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n if (event.transcript !== this.assistantText) {\n const delta = event.transcript.startsWith(this.assistantText)\n ? event.transcript.slice(this.assistantText.length)\n : event.transcript;\n\n this.assistantText = event.transcript;\n if (delta) {\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: delta,\n });\n }\n }\n return;\n\n case \"conversation.item.input_audio_transcription.completed\":\n if (event.transcript) {\n this.emitEvent({\n type: \"user.transcript\",\n text: event.transcript,\n });\n }\n return;\n\n case \"response.function_call_arguments.done\":\n this.handleToolCallDone(event);\n return;\n\n case \"error\":\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.emitEvent({\n type: \"error\",\n error: new Error(event.error?.message || \"Unknown error\"),\n });\n return;\n\n default:\n return;\n }\n }\n\n private handleToolCallDone(event: ServerEvent): void {\n const callId = event.call_id || event.item?.call_id || event.item_id;\n const name = event.name || event.item?.name;\n const argsJson = event.arguments || event.item?.arguments;\n\n if (!name || !argsJson) {\n console.warn(\"⚠️ Tool call done but missing name or arguments\");\n return;\n }\n\n let args: Record<string, unknown> = {};\n try {\n const parsed = JSON.parse(argsJson) as unknown;\n if (isRecord(parsed)) {\n args = parsed;\n }\n } catch (error) {\n console.error(\"Failed to parse tool call args\", error, argsJson);\n return;\n }\n\n this.emitEvent({\n type: \"tool.call\",\n name,\n args,\n callId,\n });\n }\n\n private async getSessionBootstrap(\n request: RealtimeSessionRequest,\n ): Promise<RealtimeSessionBootstrap> {\n if (this.options.sessionBootstrap) {\n return this.options.sessionBootstrap(request);\n }\n\n const apiEndpoint = this.options.apiEndpoint;\n if (!apiEndpoint) {\n throw new Error(\n \"OpenAI realtime client requires apiEndpoint or sessionBootstrap\",\n );\n }\n\n const response = await fetchWithTimeout(\n apiEndpoint,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(request),\n },\n `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,\n );\n\n if (!response.ok) {\n const errorText = await response.text();\n throw new Error(`Failed to create Realtime session: ${errorText}`);\n }\n\n const bootstrap = (await response.json()) as unknown;\n if (!isRealtimeSessionBootstrap(bootstrap)) {\n throw new Error(\"Invalid realtime session bootstrap response\");\n }\n\n return bootstrap as RealtimeSessionBootstrap;\n }\n\n private setupAudioAnalysis(stream: MediaStream): void {\n try {\n const audioContextConstructor =\n window.AudioContext ||\n (window as Window & { webkitAudioContext?: typeof AudioContext })\n .webkitAudioContext;\n if (!audioContextConstructor) {\n throw new Error(\"AudioContext is not supported in this browser\");\n }\n\n this.audioContext = new audioContextConstructor();\n const source = this.audioContext.createMediaStreamSource(stream);\n this.analyser = this.audioContext.createAnalyser();\n this.analyser.fftSize = 256;\n this.analyser.smoothingTimeConstant = 0.8;\n source.connect(this.analyser);\n\n this.startLipSyncAnalysis();\n } catch (error) {\n console.error(\"Failed to setup audio analysis:\", error);\n }\n }\n\n private startLipSyncAnalysis(): void {\n if (!this.analyser) {\n return;\n }\n\n const bufferLength = this.analyser.frequencyBinCount;\n const dataArray = new Uint8Array(bufferLength);\n\n this.lipSyncInterval = window.setInterval(() => {\n if (!this.analyser) {\n return;\n }\n\n this.analyser.getByteFrequencyData(dataArray);\n\n let sum = 0;\n for (let index = 0; index < bufferLength; index += 1) {\n const normalized = dataArray[index] / 255;\n sum += normalized * normalized;\n }\n\n const rms = Math.sqrt(sum / bufferLength);\n this.emitEvent({\n type: \"audio.lipsync\",\n rms: Math.min(rms * 3, 1),\n });\n }, 1000 / 60);\n }\n\n private stopLipSyncAnalysis(): void {\n if (this.lipSyncInterval) {\n clearInterval(this.lipSyncInterval);\n this.lipSyncInterval = null;\n }\n\n this.emitEvent({ type: \"audio.lipsync\", rms: 0 });\n }\n\n private cleanup(): void {\n this.stopLipSyncAnalysis();\n\n if (this.dc) {\n this.dc.close();\n this.dc = null;\n }\n\n if (this.pc) {\n this.pc.close();\n this.pc = null;\n }\n\n if (this.mediaStream) {\n this.mediaStream.getTracks().forEach((track) => track.stop());\n this.mediaStream = null;\n }\n\n if (this.audioElement) {\n this.audioElement.srcObject = null;\n this.audioElement = null;\n }\n\n if (this.audioContext) {\n this.audioContext.close();\n this.audioContext = null;\n }\n\n this.analyser = null;\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.resetResponseTracking();\n }\n\n private beginResponseRequest(): void {\n this.resetResponseTracking();\n this.isResponseInProgress = true;\n this.cancelInFlight = false;\n }\n\n private requireOpenDataChannel(): RTCDataChannel {\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n return this.dc;\n }\n\n private resetResponseTracking(): void {\n this.assistantText = \"\";\n this.hasStartedAssistantResponse = false;\n this.hasStartedAudioOutput = false;\n }\n\n private emitEvent(event: RealtimeTransportEvent): void {\n for (const callback of this.eventCallbacks) {\n callback(event);\n }\n }\n\n private log(...args: unknown[]): void {\n if (this.options.debug) {\n console.log(...args);\n }\n }\n\n private shouldLogDebugEvent(eventType: string): boolean {\n if (DEBUG_EVENT_ALLOWLIST.has(eventType)) {\n return true;\n }\n\n return eventType.startsWith(\"response.output_text.\");\n }\n}\n\nexport function createOpenAIRealtimeClient(\n options?: OpenAIRealtimeClientOptions,\n): RealtimeTransportClient {\n return new OpenAIRealtimeClient(options);\n}\n"]}
|