@charivo/realtime 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -7
- package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
- package/dist/chunk-DHNRDHST.mjs.map +1 -0
- package/dist/chunk-PDODY2T7.mjs +65 -0
- package/dist/chunk-PDODY2T7.mjs.map +1 -0
- package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
- package/dist/chunk-PTFZ7CML.mjs.map +1 -0
- package/dist/chunk-Y7JO2FFX.mjs +121 -0
- package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
- package/dist/index.d.mts +16 -4
- package/dist/index.d.ts +16 -4
- package/dist/index.js +240 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +238 -98
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +37 -1
- package/dist/openai/index.d.ts +37 -1
- package/dist/openai/index.js +427 -14
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.d.mts +26 -1
- package/dist/openai-agents/index.d.ts +26 -1
- package/dist/openai-agents/index.js +311 -18
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -2
- package/dist/remote/index.d.mts +3 -1
- package/dist/remote/index.d.ts +3 -1
- package/dist/remote/index.js +663 -32
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +10 -3
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
- package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
- package/package.json +3 -3
- package/dist/chunk-IIKECRXN.mjs.map +0 -1
- package/dist/chunk-JGJZKKMH.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -56,6 +56,11 @@ await manager.updateSession({
|
|
|
56
56
|
});
|
|
57
57
|
```
|
|
58
58
|
|
|
59
|
+
If the live transport drops temporarily, `RealtimeManager` now keeps the
|
|
60
|
+
session active and drives reconnect attempts internally. During that window
|
|
61
|
+
`state.session.status` stays `"active"` while `state.connection` moves back to
|
|
62
|
+
`"connecting"`.
|
|
63
|
+
|
|
59
64
|
## Exports
|
|
60
65
|
|
|
61
66
|
- `createRealtimeManager(client, options?)`
|
|
@@ -106,20 +111,40 @@ loaded Live2D model catalog:
|
|
|
106
111
|
|
|
107
112
|
## Session Refresh
|
|
108
113
|
|
|
109
|
-
`updateSession(...)`
|
|
110
|
-
|
|
114
|
+
`updateSession(...)` patches the active provider session in place using the
|
|
115
|
+
latest requested config, current character, and current tool registry.
|
|
111
116
|
|
|
112
117
|
- inactive managers only cache the requested base config for the next
|
|
113
118
|
`startSession(...)`
|
|
114
|
-
- active managers
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
-
|
|
119
|
+
- active managers keep the current connection open and issue a transport-level
|
|
120
|
+
session patch
|
|
121
|
+
- successful patches update `realtime:state` only and do not emit synthetic
|
|
122
|
+
`realtime:session:end/start` refresh boundaries
|
|
123
|
+
- patch failures keep the current live session and previous
|
|
124
|
+
`state.session.config` in place
|
|
125
|
+
- `state.session.config` is only replaced after the patch succeeds
|
|
119
126
|
- repeated `updateSession(...)` calls are coalesced to the latest config
|
|
120
127
|
- `stopSession()` wins over an in-flight refresh and converges to a stopped
|
|
121
128
|
session
|
|
122
129
|
|
|
130
|
+
## Reconnect Semantics
|
|
131
|
+
|
|
132
|
+
Successful reconnects are treated as a continuation of the same live session.
|
|
133
|
+
|
|
134
|
+
- successful recovery does not emit synthetic `realtime:session:end/start`
|
|
135
|
+
- `updateSession(...)` still updates the cached base config while reconnecting
|
|
136
|
+
- `sendMessage(...)`, `sendAudioChunk(...)`, `interrupt()`, and transport-level
|
|
137
|
+
tool results reject while `connection === "connecting"`
|
|
138
|
+
- the next reconnect attempt always rebuilds from the latest effective config
|
|
139
|
+
- in-flight assistant responses are marked as interrupted and are not resumed
|
|
140
|
+
- old tool-call ids are not replayed after reconnect
|
|
141
|
+
|
|
142
|
+
Observability events emitted by the manager:
|
|
143
|
+
|
|
144
|
+
- `realtime:reconnect:attempt`
|
|
145
|
+
- `realtime:reconnect:success`
|
|
146
|
+
- `realtime:reconnect:exhausted`
|
|
147
|
+
|
|
123
148
|
## Tool Registry
|
|
124
149
|
|
|
125
150
|
`RealtimeManager` owns the tool registry. Definitions sent to the provider come
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-
|
|
1
|
+
import { createIceDisconnectDebouncer, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, acquireMicrophoneStream, isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap, bindTransportLifecycle, replaceMicrophoneTrack } from './chunk-Y7JO2FFX.mjs';
|
|
2
|
+
import { DEFAULT_REALTIME_VOICE } from './chunk-PDODY2T7.mjs';
|
|
2
3
|
import { OPENAI_REALTIME_ADAPTER } from '@charivo/core';
|
|
3
4
|
|
|
4
5
|
var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
@@ -10,6 +11,7 @@ var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
|
10
11
|
"conversation.item.input_audio_transcription.completed",
|
|
11
12
|
"error"
|
|
12
13
|
]);
|
|
14
|
+
var DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5e3;
|
|
13
15
|
var OpenAIRealtimeClient = class {
|
|
14
16
|
constructor(options = {}) {
|
|
15
17
|
this.options = options;
|
|
@@ -18,7 +20,9 @@ var OpenAIRealtimeClient = class {
|
|
|
18
20
|
dc = null;
|
|
19
21
|
audioElement = null;
|
|
20
22
|
mediaStream = null;
|
|
23
|
+
audioSender = null;
|
|
21
24
|
audioContext = null;
|
|
25
|
+
audioSource = null;
|
|
22
26
|
analyser = null;
|
|
23
27
|
lipSyncInterval = null;
|
|
24
28
|
isResponseInProgress = false;
|
|
@@ -26,13 +30,35 @@ var OpenAIRealtimeClient = class {
|
|
|
26
30
|
hasStartedAssistantResponse = false;
|
|
27
31
|
hasStartedAudioOutput = false;
|
|
28
32
|
assistantText = "";
|
|
33
|
+
nextSessionUpdateId = 1;
|
|
34
|
+
pendingSessionUpdate = null;
|
|
35
|
+
currentSessionConfig;
|
|
36
|
+
connectionLossNotified = false;
|
|
37
|
+
isExplicitDisconnect = false;
|
|
38
|
+
isRecovering = false;
|
|
39
|
+
isCleaningUp = false;
|
|
40
|
+
teardownTransportLifecycle;
|
|
41
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
42
|
+
this.emitConnectionLost("ice-disconnected");
|
|
43
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
29
44
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
30
45
|
async connect(config) {
|
|
31
46
|
try {
|
|
47
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
48
|
+
throw new Error("Realtime session disconnect requested");
|
|
49
|
+
}
|
|
32
50
|
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
51
|
+
this.isExplicitDisconnect = false;
|
|
52
|
+
this.isRecovering = false;
|
|
53
|
+
this.connectionLossNotified = false;
|
|
54
|
+
this.currentSessionConfig = config;
|
|
33
55
|
this.pc = new RTCPeerConnection();
|
|
34
56
|
this.audioElement = document.createElement("audio");
|
|
35
57
|
this.audioElement.autoplay = true;
|
|
58
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
59
|
+
this.prepareAudioAnalysis();
|
|
60
|
+
this.bindConnectionEvents();
|
|
61
|
+
this.bindTransportLifecycleEvents();
|
|
36
62
|
this.pc.ontrack = (event) => {
|
|
37
63
|
if (this.audioElement) {
|
|
38
64
|
this.audioElement.srcObject = event.streams[0];
|
|
@@ -40,11 +66,9 @@ var OpenAIRealtimeClient = class {
|
|
|
40
66
|
this.setupAudioAnalysis(event.streams[0]);
|
|
41
67
|
};
|
|
42
68
|
try {
|
|
43
|
-
this.mediaStream = await
|
|
44
|
-
audio: true
|
|
45
|
-
});
|
|
69
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
46
70
|
const audioTrack = this.mediaStream.getTracks()[0];
|
|
47
|
-
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
71
|
+
this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
|
|
48
72
|
} catch {
|
|
49
73
|
throw new Error("Microphone access required for Realtime API");
|
|
50
74
|
}
|
|
@@ -83,9 +107,64 @@ var OpenAIRealtimeClient = class {
|
|
|
83
107
|
}
|
|
84
108
|
async disconnect() {
|
|
85
109
|
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
110
|
+
this.isExplicitDisconnect = true;
|
|
111
|
+
this.isRecovering = false;
|
|
86
112
|
this.cleanup();
|
|
87
113
|
}
|
|
114
|
+
async updateSession(config) {
|
|
115
|
+
if (this.isResponseInProgress) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
"Cannot update the realtime session while a response is in progress. Call interrupt() first."
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
const dc = this.requireOpenDataChannel();
|
|
121
|
+
const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;
|
|
122
|
+
if (this.pendingSessionUpdate) {
|
|
123
|
+
throw new Error("Realtime session update already in progress");
|
|
124
|
+
}
|
|
125
|
+
dc.send(
|
|
126
|
+
JSON.stringify({
|
|
127
|
+
type: "session.update",
|
|
128
|
+
event_id: eventId,
|
|
129
|
+
session: toOpenAIRealtimeSessionUpdate(config)
|
|
130
|
+
})
|
|
131
|
+
);
|
|
132
|
+
await new Promise((resolve, reject) => {
|
|
133
|
+
const timeoutId = setTimeout(() => {
|
|
134
|
+
this.rejectPendingSessionUpdate(
|
|
135
|
+
new Error(
|
|
136
|
+
`Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`
|
|
137
|
+
)
|
|
138
|
+
);
|
|
139
|
+
}, this.getSessionUpdateTimeoutMs());
|
|
140
|
+
this.pendingSessionUpdate = {
|
|
141
|
+
eventId,
|
|
142
|
+
resolve,
|
|
143
|
+
reject,
|
|
144
|
+
timeoutId
|
|
145
|
+
};
|
|
146
|
+
});
|
|
147
|
+
this.currentSessionConfig = config;
|
|
148
|
+
}
|
|
149
|
+
async recover(config) {
|
|
150
|
+
this.isRecovering = true;
|
|
151
|
+
this.connectionLossNotified = true;
|
|
152
|
+
this.currentSessionConfig = config;
|
|
153
|
+
try {
|
|
154
|
+
if (await this.tryIceRestartRecovery(config)) {
|
|
155
|
+
this.connectionLossNotified = false;
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
await this.rebuildConnection(config);
|
|
159
|
+
this.connectionLossNotified = false;
|
|
160
|
+
} finally {
|
|
161
|
+
this.isRecovering = false;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
88
164
|
async sendText(text) {
|
|
165
|
+
if (this.isRecovering) {
|
|
166
|
+
throw new Error("Realtime transport reconnecting");
|
|
167
|
+
}
|
|
89
168
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
90
169
|
throw new Error("DataChannel not ready");
|
|
91
170
|
}
|
|
@@ -112,11 +191,17 @@ var OpenAIRealtimeClient = class {
|
|
|
112
191
|
this.beginResponseRequest();
|
|
113
192
|
}
|
|
114
193
|
async sendAudio(_audio) {
|
|
194
|
+
if (this.isRecovering) {
|
|
195
|
+
throw new Error("Realtime transport reconnecting");
|
|
196
|
+
}
|
|
115
197
|
console.warn(
|
|
116
198
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
117
199
|
);
|
|
118
200
|
}
|
|
119
201
|
async sendToolResult(callId, output) {
|
|
202
|
+
if (this.isRecovering) {
|
|
203
|
+
throw new Error("Realtime transport reconnecting");
|
|
204
|
+
}
|
|
120
205
|
const dc = this.requireOpenDataChannel();
|
|
121
206
|
dc.send(
|
|
122
207
|
JSON.stringify({
|
|
@@ -132,6 +217,9 @@ var OpenAIRealtimeClient = class {
|
|
|
132
217
|
this.beginResponseRequest();
|
|
133
218
|
}
|
|
134
219
|
async interrupt() {
|
|
220
|
+
if (this.isRecovering) {
|
|
221
|
+
throw new Error("Realtime transport reconnecting");
|
|
222
|
+
}
|
|
135
223
|
const dc = this.requireOpenDataChannel();
|
|
136
224
|
if (!this.isResponseInProgress) {
|
|
137
225
|
return;
|
|
@@ -149,7 +237,9 @@ var OpenAIRealtimeClient = class {
|
|
|
149
237
|
}
|
|
150
238
|
switch (event.type) {
|
|
151
239
|
case "session.created":
|
|
240
|
+
return;
|
|
152
241
|
case "session.updated":
|
|
242
|
+
this.resolvePendingSessionUpdate();
|
|
153
243
|
return;
|
|
154
244
|
case "response.audio.delta":
|
|
155
245
|
case "output_audio_buffer.started":
|
|
@@ -255,10 +345,16 @@ var OpenAIRealtimeClient = class {
|
|
|
255
345
|
case "error":
|
|
256
346
|
this.isResponseInProgress = false;
|
|
257
347
|
this.cancelInFlight = false;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
348
|
+
{
|
|
349
|
+
const error = new Error(event.error?.message || "Unknown error");
|
|
350
|
+
if (this.isPendingSessionUpdateError(event)) {
|
|
351
|
+
this.rejectPendingSessionUpdate(error);
|
|
352
|
+
}
|
|
353
|
+
this.emitEvent({
|
|
354
|
+
type: "error",
|
|
355
|
+
error
|
|
356
|
+
});
|
|
357
|
+
}
|
|
262
358
|
return;
|
|
263
359
|
default:
|
|
264
360
|
return;
|
|
@@ -320,18 +416,133 @@ var OpenAIRealtimeClient = class {
|
|
|
320
416
|
}
|
|
321
417
|
return bootstrap;
|
|
322
418
|
}
|
|
419
|
+
bindConnectionEvents() {
|
|
420
|
+
if (!this.pc) {
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
424
|
+
const iceState = this.pc?.iceConnectionState;
|
|
425
|
+
if (iceState === "failed") {
|
|
426
|
+
this.iceDisconnectDebouncer.cancel();
|
|
427
|
+
this.emitConnectionLost("ice-failed");
|
|
428
|
+
} else if (iceState === "disconnected") {
|
|
429
|
+
this.iceDisconnectDebouncer.schedule();
|
|
430
|
+
} else {
|
|
431
|
+
this.iceDisconnectDebouncer.cancel();
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
435
|
+
const connectionState = this.pc?.connectionState;
|
|
436
|
+
if (connectionState === "failed" || connectionState === "closed") {
|
|
437
|
+
this.iceDisconnectDebouncer.cancel();
|
|
438
|
+
this.emitConnectionLost("connection-failed");
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
bindTransportLifecycleEvents() {
|
|
443
|
+
if (this.teardownTransportLifecycle) {
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
447
|
+
onHidden: this.handleHidden,
|
|
448
|
+
onOnline: this.handleOnline,
|
|
449
|
+
onPageHide: this.handlePageHide,
|
|
450
|
+
onPageShow: this.handlePageShow,
|
|
451
|
+
onVisible: this.handleVisible,
|
|
452
|
+
onDeviceChange: this.handleDeviceChange
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
unbindTransportLifecycleEvents() {
|
|
456
|
+
this.teardownTransportLifecycle?.();
|
|
457
|
+
this.teardownTransportLifecycle = void 0;
|
|
458
|
+
}
|
|
459
|
+
handleOnline = () => {
|
|
460
|
+
this.emitConnectionLost("online");
|
|
461
|
+
};
|
|
462
|
+
handleHidden = () => {
|
|
463
|
+
this.pauseAudioAnalysis();
|
|
464
|
+
};
|
|
465
|
+
handleVisible = () => {
|
|
466
|
+
this.resumeAudioAnalysis();
|
|
467
|
+
this.emitConnectionLost("visibility");
|
|
468
|
+
};
|
|
469
|
+
handlePageHide = () => {
|
|
470
|
+
this.pauseAudioAnalysis();
|
|
471
|
+
};
|
|
472
|
+
handlePageShow = (event) => {
|
|
473
|
+
this.resumeAudioAnalysis();
|
|
474
|
+
if (event.persisted) {
|
|
475
|
+
this.emitConnectionLost("pageshow");
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
handleDeviceChange = () => {
|
|
479
|
+
void this.refreshMicrophoneTrack();
|
|
480
|
+
};
|
|
481
|
+
async refreshMicrophoneTrack() {
|
|
482
|
+
if (!this.pc || this.isExplicitDisconnect) {
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
try {
|
|
486
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
487
|
+
audioSender: this.audioSender,
|
|
488
|
+
mediaStream: this.mediaStream,
|
|
489
|
+
peerConnection: this.pc
|
|
490
|
+
});
|
|
491
|
+
this.audioSender = audioSender;
|
|
492
|
+
this.mediaStream = mediaStream;
|
|
493
|
+
} catch (error) {
|
|
494
|
+
this.emitEvent({
|
|
495
|
+
type: "error",
|
|
496
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
emitConnectionLost(cause, error) {
|
|
501
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
this.connectionLossNotified = true;
|
|
505
|
+
this.isResponseInProgress = false;
|
|
506
|
+
this.cancelInFlight = false;
|
|
507
|
+
this.resetResponseTracking();
|
|
508
|
+
this.stopLipSyncAnalysis();
|
|
509
|
+
this.emitEvent({
|
|
510
|
+
type: "connection.lost",
|
|
511
|
+
cause,
|
|
512
|
+
error
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
needsRecovery(cause) {
|
|
516
|
+
if (!this.pc || !this.dc) {
|
|
517
|
+
return false;
|
|
518
|
+
}
|
|
519
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
520
|
+
return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
|
|
521
|
+
}
|
|
522
|
+
return true;
|
|
523
|
+
}
|
|
524
|
+
prepareAudioAnalysis() {
|
|
525
|
+
if (this.audioContext) {
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
529
|
+
if (!audioContextConstructor) {
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
this.audioContext = new audioContextConstructor();
|
|
533
|
+
}
|
|
323
534
|
setupAudioAnalysis(stream) {
|
|
324
535
|
try {
|
|
325
|
-
|
|
326
|
-
if (!
|
|
327
|
-
|
|
536
|
+
this.prepareAudioAnalysis();
|
|
537
|
+
if (!this.audioContext) {
|
|
538
|
+
return;
|
|
328
539
|
}
|
|
329
|
-
this.
|
|
330
|
-
|
|
540
|
+
this.audioSource?.disconnect();
|
|
541
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
331
542
|
this.analyser = this.audioContext.createAnalyser();
|
|
332
543
|
this.analyser.fftSize = 256;
|
|
333
544
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
334
|
-
|
|
545
|
+
this.audioSource.connect(this.analyser);
|
|
335
546
|
this.startLipSyncAnalysis();
|
|
336
547
|
} catch (error) {
|
|
337
548
|
console.error("Failed to setup audio analysis:", error);
|
|
@@ -367,8 +578,23 @@ var OpenAIRealtimeClient = class {
|
|
|
367
578
|
}
|
|
368
579
|
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
369
580
|
}
|
|
581
|
+
pauseAudioAnalysis() {
|
|
582
|
+
this.stopLipSyncAnalysis();
|
|
583
|
+
}
|
|
584
|
+
resumeAudioAnalysis() {
|
|
585
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
this.startLipSyncAnalysis();
|
|
589
|
+
}
|
|
370
590
|
cleanup() {
|
|
591
|
+
this.isCleaningUp = true;
|
|
592
|
+
this.unbindTransportLifecycleEvents();
|
|
593
|
+
this.iceDisconnectDebouncer.cancel();
|
|
371
594
|
this.stopLipSyncAnalysis();
|
|
595
|
+
this.rejectPendingSessionUpdate(
|
|
596
|
+
new Error("Realtime session ended before session update completed")
|
|
597
|
+
);
|
|
372
598
|
if (this.dc) {
|
|
373
599
|
this.dc.close();
|
|
374
600
|
this.dc = null;
|
|
@@ -389,9 +615,13 @@ var OpenAIRealtimeClient = class {
|
|
|
389
615
|
this.audioContext.close();
|
|
390
616
|
this.audioContext = null;
|
|
391
617
|
}
|
|
618
|
+
this.audioSource = null;
|
|
619
|
+
this.audioSender = null;
|
|
392
620
|
this.analyser = null;
|
|
393
621
|
this.isResponseInProgress = false;
|
|
394
622
|
this.cancelInFlight = false;
|
|
623
|
+
this.connectionLossNotified = false;
|
|
624
|
+
this.isCleaningUp = false;
|
|
395
625
|
this.resetResponseTracking();
|
|
396
626
|
}
|
|
397
627
|
beginResponseRequest() {
|
|
@@ -400,11 +630,86 @@ var OpenAIRealtimeClient = class {
|
|
|
400
630
|
this.cancelInFlight = false;
|
|
401
631
|
}
|
|
402
632
|
requireOpenDataChannel() {
|
|
633
|
+
if (this.isRecovering) {
|
|
634
|
+
throw new Error("Realtime transport reconnecting");
|
|
635
|
+
}
|
|
403
636
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
404
637
|
throw new Error("DataChannel not ready");
|
|
405
638
|
}
|
|
406
639
|
return this.dc;
|
|
407
640
|
}
|
|
641
|
+
async tryIceRestartRecovery(config) {
|
|
642
|
+
const pc = this.pc;
|
|
643
|
+
if (!pc || this.shouldRebuildConnection()) {
|
|
644
|
+
return false;
|
|
645
|
+
}
|
|
646
|
+
if (pc.iceConnectionState === "connected") {
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
649
|
+
if (pc.iceConnectionState === "disconnected") {
|
|
650
|
+
await delay(1500);
|
|
651
|
+
if (pc.iceConnectionState !== "disconnected") {
|
|
652
|
+
return true;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
656
|
+
return false;
|
|
657
|
+
}
|
|
658
|
+
if (typeof pc.restartIce === "function") {
|
|
659
|
+
pc.restartIce();
|
|
660
|
+
}
|
|
661
|
+
const offer = await pc.createOffer({ iceRestart: true });
|
|
662
|
+
await pc.setLocalDescription(offer);
|
|
663
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
664
|
+
transport: "webrtc",
|
|
665
|
+
session: config ?? this.currentSessionConfig ?? {},
|
|
666
|
+
sdpOffer: offer.sdp
|
|
667
|
+
});
|
|
668
|
+
if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
669
|
+
throw new Error(
|
|
670
|
+
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
671
|
+
);
|
|
672
|
+
}
|
|
673
|
+
await pc.setRemoteDescription({
|
|
674
|
+
type: "answer",
|
|
675
|
+
sdp: bootstrap.answerSdp
|
|
676
|
+
});
|
|
677
|
+
return true;
|
|
678
|
+
}
|
|
679
|
+
shouldRebuildConnection() {
|
|
680
|
+
return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
|
|
681
|
+
}
|
|
682
|
+
async rebuildConnection(config) {
|
|
683
|
+
this.connectionLossNotified = true;
|
|
684
|
+
this.cleanup();
|
|
685
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
686
|
+
}
|
|
687
|
+
resolvePendingSessionUpdate() {
|
|
688
|
+
if (!this.pendingSessionUpdate) {
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
692
|
+
this.pendingSessionUpdate.resolve();
|
|
693
|
+
this.pendingSessionUpdate = null;
|
|
694
|
+
}
|
|
695
|
+
rejectPendingSessionUpdate(error) {
|
|
696
|
+
if (!this.pendingSessionUpdate) {
|
|
697
|
+
return;
|
|
698
|
+
}
|
|
699
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
700
|
+
this.pendingSessionUpdate.reject(error);
|
|
701
|
+
this.pendingSessionUpdate = null;
|
|
702
|
+
}
|
|
703
|
+
isPendingSessionUpdateError(event) {
|
|
704
|
+
const pendingUpdate = this.pendingSessionUpdate;
|
|
705
|
+
if (!pendingUpdate) {
|
|
706
|
+
return false;
|
|
707
|
+
}
|
|
708
|
+
return event.error?.event_id === pendingUpdate.eventId;
|
|
709
|
+
}
|
|
710
|
+
getSessionUpdateTimeoutMs() {
|
|
711
|
+
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
712
|
+
}
|
|
408
713
|
resetResponseTracking() {
|
|
409
714
|
this.assistantText = "";
|
|
410
715
|
this.hasStartedAssistantResponse = false;
|
|
@@ -430,7 +735,35 @@ var OpenAIRealtimeClient = class {
|
|
|
430
735
|
function createOpenAIRealtimeClient(options) {
|
|
431
736
|
return new OpenAIRealtimeClient(options);
|
|
432
737
|
}
|
|
738
|
+
function toOpenAIRealtimeSessionUpdate(config) {
|
|
739
|
+
const session = {
|
|
740
|
+
audio: {
|
|
741
|
+
output: {
|
|
742
|
+
voice: config?.voice ?? DEFAULT_REALTIME_VOICE
|
|
743
|
+
}
|
|
744
|
+
},
|
|
745
|
+
tool_choice: config?.toolChoice ?? "auto"
|
|
746
|
+
};
|
|
747
|
+
if (config?.instructions) {
|
|
748
|
+
session.instructions = config.instructions;
|
|
749
|
+
}
|
|
750
|
+
if (config?.temperature !== void 0) {
|
|
751
|
+
session.temperature = config.temperature;
|
|
752
|
+
}
|
|
753
|
+
if (config?.maxTokens !== void 0) {
|
|
754
|
+
session.max_response_output_tokens = config.maxTokens;
|
|
755
|
+
}
|
|
756
|
+
if (config?.tools?.length) {
|
|
757
|
+
session.tools = config.tools;
|
|
758
|
+
}
|
|
759
|
+
return session;
|
|
760
|
+
}
|
|
761
|
+
function delay(ms) {
|
|
762
|
+
return new Promise((resolve) => {
|
|
763
|
+
setTimeout(resolve, ms);
|
|
764
|
+
});
|
|
765
|
+
}
|
|
433
766
|
|
|
434
767
|
export { OpenAIRealtimeClient, createOpenAIRealtimeClient };
|
|
435
|
-
//# sourceMappingURL=chunk-
|
|
436
|
-
//# sourceMappingURL=chunk-
|
|
768
|
+
//# sourceMappingURL=chunk-DHNRDHST.mjs.map
|
|
769
|
+
//# sourceMappingURL=chunk-DHNRDHST.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/openai/client.ts"],"names":[],"mappings":";;;;AAiEA,IAAM,qBAAA,uBAA4B,GAAA,CAAY;AAAA,EAC5C,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,kBAAA;AAAA,EACA,eAAA;AAAA,EACA,uCAAA;AAAA,EACA,uDAAA;AAAA,EACA;AACF,CAAC,CAAA;AACD,IAAM,iCAAA,GAAoC,GAAA;AAQnC,IAAM,uBAAN,MAA8D;AAAA,EA4BnE,WAAA,CAAoB,OAAA,GAAuC,EAAC,EAAG;AAA3C,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAA4C;AAAA,EA3BxD,EAAA,GAA+B,IAAA;AAAA,EAC/B,EAAA,GAA4B,IAAA;AAAA,EAC5B,YAAA,GAAwC,IAAA;AAAA,EACxC,WAAA,GAAkC,IAAA;AAAA,EAClC,WAAA,GAAmC,IAAA;AAAA,EACnC,YAAA,GAAoC,IAAA;AAAA,EACpC,WAAA,GAAiD,IAAA;AAAA,EACjD,QAAA,GAAgC,IAAA;AAAA,EAChC,eAAA,GAAiC,IAAA;AAAA,EACjC,oBAAA,GAAuB,KAAA;AAAA,EACvB,cAAA,GAAiB,KAAA;AAAA,EACjB,2BAAA,GAA8B,KAAA;AAAA,EAC9B,qBAAA,GAAwB,KAAA;AAAA,EACxB,aAAA,GAAgB,EAAA;AAAA,EAChB,mBAAA,GAAsB,CAAA;AAAA,EACtB,oBAAA,GAAoD,IAAA;AAAA,EACpD,oBAAA;AAAA,EACA,sBAAA,GAAyB,KAAA;AAAA,EACzB,oBAAA,GAAuB,KAAA;AAAA,EACvB,YAAA,GAAe,KAAA;AAAA,EACf,YAAA,GAAe,KAAA;AAAA,EACf,0BAAA;AAAA,EACS,sBAAA,GAAyB,6BAA6B,MAAM;AAC3E,IAAA,IAAA,CAAK,mBAAmB,kBAAkB,CAAA;AAAA,EAC5C,GAAG,oCAAoC,CAAA;AAAA,EAC/B,cAAA,uBAAqB,GAAA,EAA6C;AAAA,EAI1E,MAAM,QAAQ,MAAA,EAA+C;AAC3D,IAAA,IAAI;AACF,MAAA,IAAI,IAAA,CAAK,oBAAA,IAAwB,IAAA,CAAK,YAAA,EAAc;AAClD,QAAA,MAAM,IAAI,MAAM,uCAAuC,CAAA;AAAA,MACzD;AAEA,MAAA,IAAA,CAAK,IAAI,4CAA4C,CAAA;AAErD,MAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,MAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,MAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,MAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAC5B,MAAA,IAAA,CAAK,EAAA,GAAK,IAAI,iBAAA,EAAkB;AAChC,MAAA,IAAA,CAAK,YAAA,GAAe,QAAA,CAAS,aAAA,CAAc,OAAO,CAAA;AAClD,MAAA,IAAA,CAAK,aAAa,QAAA,GAAW,IAAA;AAC7B,MAAA,IAAA,CAAK,YAAA,CAAa,YAAA,CAAa,aAAA,EAAe,MAAM,CAAA;AACpD,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAC1B,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAC1B,MAAA,IAAA,CAAK,4BAAA,EAA6B;AAElC,MAAA,IAAA,CAAK,EAAA,CAAG,OAAA,GAAU,CAAC,KAAA,KAAU;AAC3B,QAAA,IAAI,KAAK,YAAA,EAAc;AACrB,UAAA,IAAA,CAAK,YAAA,CAAa,SAAA,GAAY,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAA;AAAA,QAC/C;AACA,QAAA,IAAA,CAAK,kBAAA,CAAmB,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA;AAAA,MAC1C,CAAA;AAEA,MAAA,IAAI;AACF,QAAA,IAAA,CAAK,WAAA,GAAc,MAAM,uBAAA,EAAwB;AACjD,QAAA,MAAM,UAAA,GAAa,IAAA,CAAK,WAAA,CAAY,SAAA,GAAY,CAAC,CAAA;AACjD,QAAA,IAAA,CAAK,WAAA,GACH,KAAK,EAAA,CAAG,QAAA,CAAS,YAAY,IAAA,CAAK,WAAW,KAAK,IAAA,CAAK,WAAA;AAAA,MAC3D,CAAA,CAAA,MAAQ;AACN,QAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,MAC/D;AAEA,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA,CAAK,EAAA,CAAG,iBAAA,CAAkB,YAAY,CAAA;AAChD,MAAA,IAAA,CAAK,EAAA,CAAG,SAAA,GAAY,CAAC,KAAA,KAAU;AAC7B,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,KAAA,CAAM,KAAA,CAAM,IAAI,CAAA;AACrC,QAAA,IAAA,CAAK,kBAAkB,OAAO,CAAA;AAAA,MAChC,CAAA;AACA,MAAA,IAAA,CAAK,EAAA,CAAG,UAAU,MAAM;AACtB,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,OAAA;AAAA,UACN,KAAA,EAAO,IAAI,KAAA,CAAM,mBAAmB;AAAA,SACrC,CAAA;AAAA,MACH,CAAA;AAEA,MAAA,MAAM,KAAA,GAAQ,MAAM,IAAA,CAAK,EAAA,CAAG,WAAA,EAAY;AACxC,MAAA,MAAM,IAAA,CAAK,EAAA,CAAG,mBAAA,CAAoB,KAAK,CAAA;AAEvC,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,mBAAA,CAAoB;AAAA,QAC/C,SAAA,EAAW,QAAA;AAAA,QACX,OAAA,EAAS,UAAU,EAAC;AAAA,QACpB,UAAU,KAAA,CAAM;AAAA,OACjB,CAAA;AAED,MAAA,IACE,SAAA,CAAU,YAAY,uBAAA,IACtB,SAAA,CAAU,cAAc,QAAA,IACxB,EAAE,eAAe,SAAA,CAAA,EACjB;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,wCAAwC,uBAAuB,CAAA,qBAAA,EAAwB,UAAU,OAAO,CAAA,CAAA,EAAI,UAAU,SAAS,CAAA;AAAA,SACjI;AAAA,MACF;AAEA,MAAA,MAAM,IAAA,CAAK,GAAG,oBAAA,CAAqB;AAAA,QACjC,IAAA,EAAM,QAAA;AAAA,QACN,KAAK,SAAA,CAAU;AAAA,OAChB,CAAA;AAED,MAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAA;AAAA,IAC5C,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,EAAQ;AACb,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,IAAI,sCAAsC,CAAA;AAC/C,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,IAAA,IAAA,CAAK,OAAA,EAAQ;AAAA,EACf;AAAA,EAEA,MAAM,cAAc,MAAA,EAA+C;AACjE,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AACvC,IAAA,MAAM,OAAA,GAAU,CAAA,uBAAA,EAA0B,IAAA,CAAK,mBAAA,EAAqB,CAAA,CAAA;AAEpE,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,IAC/D;AAEA,IAAA,EAAA,CAAG,IAAA;AAAA,MACD,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,gBAAA;AAAA,QACN,QAAA,EAAU,OAAA;AAAA,QACV,OAAA,EAAS,8BAA8B,MAAM;AAAA,OAC9C;AAAA,KACH;AAEA,IAAA,MAAM,IAAI,OAAA,CAAc,CAAC,OAAA,EAAS,MAAA,KAAW;AAC3C,MAAA,MAAM,SAAA,GAAY,WAAW,MAAM;AACjC,QAAA,IAAA,CAAK,0BAAA;AAAA,UACH,IAAI,KAAA;AAAA,YACF,CAAA,4CAAA,EAA+C,IAAA,CAAK,yBAAA,EAA2B,CAAA,EAAA;AAAA;AACjF,SACF;AAAA,MACF,CAAA,EAAG,IAAA,CAAK,yBAAA,EAA2B,CAAA;AAEnC,MAAA,IAAA,CAAK,oBAAA,GAAuB;AAAA,QAC1B,OAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAAA,EAC9B;AAAA,EAEA,MAAM,QAAQ,MAAA,EAA+C;AAC3D,IAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AACpB,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAE5B,IAAA,IAAI;AACF,MAAA,IAAI,MAAM,IAAA,CAAK,qBAAA,CAAsB,MAAM,CAAA,EAAG;AAC5C,QAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,IAAA,CAAK,kBAAkB,MAAM,CAAA;AACnC,MAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAAA,IAChC,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,IAAA,EAA6B;AAC1C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,OAAA,CAAQ,KAAK,6DAAmD,CAAA;AAChE,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,EAAA,CAAG,IAAA;AAAA,MACN,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,SAAA;AAAA,UACN,IAAA,EAAM,MAAA;AAAA,UACN,OAAA,EAAS;AAAA,YACP;AAAA,cACE,IAAA,EAAM,YAAA;AAAA,cACN;AAAA;AACF;AACF;AACF,OACD;AAAA,KACH;AACA,IAAA,IAAA,CAAK,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAExD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,UAAU,MAAA,EAAoC;AAClD,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN;AAAA,KACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAA,CACJ,MAAA,EACA,MAAA,EACe;AACf,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,EAAA,CAAG,IAAA;AAAA,MACD,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,sBAAA;AAAA,UACN,OAAA,EAAS,MAAA;AAAA,UACT,MAAA,EAAQ,IAAA,CAAK,SAAA,CAAU,MAAM;AAAA;AAC/B,OACD;AAAA,KACH;AACA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAEnD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,SAAA,GAA2B;AAC/B,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AACnD,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,IAAA;AAAA,EACxB;AAAA,EAEA,QAAQ,QAAA,EAAyD;AAC/D,IAAA,IAAA,CAAK,cAAA,CAAe,IAAI,QAAQ,CAAA;AAAA,EAClC;AAAA,EAEQ,kBAAkB,KAAA,EAA0B;AAClD,IAAA,IAAI,KAAK,OAAA,CAAQ,KAAA,IAAS,KAAK,mBAAA,CAAoB,KAAA,CAAM,IAAI,CAAA,EAAG;AAC9D,MAAA,IAAA,CAAK,GAAA,CAAI,mCAAA,EAA8B,KAAA,CAAM,IAAA,EAAM,KAAK,CAAA;AAAA,IAC1D;AAEA,IAAA,QAAQ,MAAM,IAAA;AAAM,MAClB,KAAK,iBAAA;AACH,QAAA;AAAA,MAEF,KAAK,iBAAA;AACH,QAAA,IAAA,CAAK,2BAAA,EAA4B;AACjC,QAAA;AAAA,MAEF,KAAK,sBAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,qBAAA,EAAuB;AAC/B,UAAA,IAAA,CAAK,qBAAA,GAAwB,IAAA;AAC7B,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,sBAAA,EAAwB,CAAA;AAAA,QACjD;AACA,QAAA;AAAA,MAEF,KAAK,qBAAA;AAAA,MACL,KAAK,4BAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,oBAAA,EAAsB,CAAA;AAC7C,QAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,eAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,UAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,UAAA;AAAA,QACF;AAOA,QAAA,IAAI,CAAC,KAAK,aAAA,EAAe;AACvB,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,8BAAA;AAAA,UACN,MAAM,IAAA,CAAK;AAAA,SACZ,CAAA;AACD,QAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,QAAA;AAAA,MAEF,KAAK,iCAAA;AAAA,MACL,KAAK,wCAAA;AAAA,MACL,KAAK,4BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,KAAA,EAAO;AACvC,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAA,CAAK,iBAAiB,KAAA,CAAM,KAAA;AAC5B,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,sBAAA;AAAA,UACN,MAAM,KAAA,CAAM;AAAA,SACb,CAAA;AACD,QAAA;AAAA,MAEF,KAAK,2BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,IAAA,EAAM;AACtC,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAI,KAAA,CAAM,IAAA,KAAS,IAAA,CAAK,aAAA,EAAe;AACrC,UAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,IAAA,CAAK,UAAA,CAAW,KAAK,aAAa,CAAA,GAClD,KAAA,CAAM,IAAA,CAAK,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,MAAM,IAC1C,KAAA,CAAM,IAAA;AAEV,UAAA,IAAA,CAAK,gBAAgB,KAAA,CAAM,IAAA;AAC3B,UAAA,IAAI,KAAA,EAAO;AACT,YAAA,IAAA,CAAK,SAAA,CAAU;AAAA,cACb,IAAA,EAAM,sBAAA;AAAA,cACN,IAAA,EAAM;AAAA,aACP,CAAA;AAAA,UACH;AAAA,QACF;AACA,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,UAAA,EAAY;AAC5C,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,UAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,QACvD;AAEA,QAAA,IAAI,KAAA,CAAM,UAAA,KAAe,IAAA,CAAK,aAAA,EAAe;AAC3C,UAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,UAAA,CAAW,UAAA,CAAW,KAAK,aAAa,CAAA,GACxD,KAAA,CAAM,UAAA,CAAW,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,MAAM,IAChD,KAAA,CAAM,UAAA;AAEV,UAAA,IAAA,CAAK,gBAAgB,KAAA,CAAM,UAAA;AAC3B,UAAA,IAAI,KAAA,EAAO;AACT,YAAA,IAAA,CAAK,SAAA,CAAU;AAAA,cACb,IAAA,EAAM,sBAAA;AAAA,cACN,IAAA,EAAM;AAAA,aACP,CAAA;AAAA,UACH;AAAA,QACF;AACA,QAAA;AAAA,MAEF,KAAK,uDAAA;AACH,QAAA,IAAI,MAAM,UAAA,EAAY;AACpB,UAAA,IAAA,CAAK,SAAA,CAAU;AAAA,YACb,IAAA,EAAM,iBAAA;AAAA,YACN,MAAM,KAAA,CAAM;AAAA,WACb,CAAA;AAAA,QACH;AACA,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAA,CAAK,mBAAmB,KAAK,CAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,OAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,QAAA;AACE,UAAA,MAAM,QAAQ,IAAI,KAAA,CAAM,KAAA,CAAM,KAAA,EAAO,WAAW,eAAe,CAAA;AAC/D,UAAA,IAAI,IAAA,CAAK,2BAAA,CAA4B,KAAK,CAAA,EAAG;AAC3C,YAAA,IAAA,CAAK,2BAA2B,KAAK,CAAA;AAAA,UACvC;AAEA,UAAA,IAAA,CAAK,SAAA,CAAU;AAAA,YACb,IAAA,EAAM,OAAA;AAAA,YACN;AAAA,WACD,CAAA;AAAA,QACH;AACA,QAAA;AAAA,MAEF;AACE,QAAA;AAAA;AACJ,EACF;AAAA,EAEQ,mBAAmB,KAAA,EAA0B;AACnD,IAAA,MAAM,SAAS,KAAA,CAAM,OAAA,IAAW,KAAA,CAAM,IAAA,EAAM,WAAW,KAAA,CAAM,OAAA;AAC7D,IAAA,MAAM,IAAA,GAAO,KAAA,CAAM,IAAA,IAAQ,KAAA,CAAM,IAAA,EAAM,IAAA;AACvC,IAAA,MAAM,QAAA,GAAW,KAAA,CAAM,SAAA,IAAa,KAAA,CAAM,IAAA,EAAM,SAAA;AAEhD,IAAA,IAAI,CAAC,IAAA,IAAQ,CAAC,QAAA,EAAU;AACtB,MAAA,OAAA,CAAQ,KAAK,2DAAiD,CAAA;AAC9D,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,OAAgC,EAAC;AACrC,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,QAAQ,CAAA;AAClC,MAAA,IAAI,QAAA,CAAS,MAAM,CAAA,EAAG;AACpB,QAAA,IAAA,GAAO,MAAA;AAAA,MACT;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,gCAAA,EAAkC,KAAA,EAAO,QAAQ,CAAA;AAC/D,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,SAAA,CAAU;AAAA,MACb,IAAA,EAAM,WAAA;AAAA,MACN,IAAA;AAAA,MACA,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,oBACZ,OAAA,EACmC;AACnC,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,OAAO,IAAA,CAAK,OAAA,CAAQ,gBAAA,CAAiB,OAAO,CAAA;AAAA,IAC9C;AAEA,IAAA,MAAM,WAAA,GAAc,KAAK,OAAA,CAAQ,WAAA;AACjC,IAAA,IAAI,CAAC,WAAA,EAAa;AAChB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,MAAM,WAAW,MAAM,gBAAA;AAAA,MACrB,WAAA;AAAA,MACA;AAAA,QACE,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB;AAAA,SAClB;AAAA,QACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,OAAO;AAAA,OAC9B;AAAA,MACA,4CAA4C,0BAA0B,CAAA,EAAA;AAAA,KACxE;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mCAAA,EAAsC,SAAS,CAAA,CAAE,CAAA;AAAA,IACnE;AAEA,IAAA,MAAM,SAAA,GAAa,MAAM,QAAA,CAAS,IAAA,EAAK;AACvC,IAAA,IAAI,CAAC,0BAAA,CAA2B,SAAS,CAAA,EAAG;AAC1C,MAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,IAC/D;AAEA,IAAA,OAAO,SAAA;AAAA,EACT;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,EAAA,EAAI;AACZ,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,EAAA,CAAG,gBAAA,CAAiB,0BAAA,EAA4B,MAAM;AACzD,MAAA,MAAM,QAAA,GAAW,KAAK,EAAA,EAAI,kBAAA;AAC1B,MAAA,IAAI,aAAa,QAAA,EAAU;AACzB,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,QAAA,IAAA,CAAK,mBAAmB,YAAY,CAAA;AAAA,MACtC,CAAA,MAAA,IAAW,aAAa,cAAA,EAAgB;AACtC,QAAA,IAAA,CAAK,uBAAuB,QAAA,EAAS;AAAA,MACvC,CAAA,MAAO;AACL,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AAAA,MACrC;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,EAAA,CAAG,gBAAA,CAAiB,uBAAA,EAAyB,MAAM;AACtD,MAAA,MAAM,eAAA,GAAkB,KAAK,EAAA,EAAI,eAAA;AACjC,MAAA,IAAI,eAAA,KAAoB,QAAA,IAAY,eAAA,KAAoB,QAAA,EAAU;AAChE,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,QAAA,IAAA,CAAK,mBAAmB,mBAAmB,CAAA;AAAA,MAC7C;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA,EAEQ,4BAAA,GAAqC;AAC3C,IAAA,IAAI,KAAK,0BAAA,EAA4B;AACnC,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,6BAA6B,sBAAA,CAAuB;AAAA,MACvD,UAAU,IAAA,CAAK,YAAA;AAAA,MACf,UAAU,IAAA,CAAK,YAAA;AAAA,MACf,YAAY,IAAA,CAAK,cAAA;AAAA,MACjB,YAAY,IAAA,CAAK,cAAA;AAAA,MACjB,WAAW,IAAA,CAAK,aAAA;AAAA,MAChB,gBAAgB,IAAA,CAAK;AAAA,KACtB,CAAA;AAAA,EACH;AAAA,EAEQ,8BAAA,GAAuC;AAC7C,IAAA,IAAA,CAAK,0BAAA,IAA6B;AAClC,IAAA,IAAA,CAAK,0BAAA,GAA6B,MAAA;AAAA,EACpC;AAAA,EAEiB,eAAe,MAAY;AAC1C,IAAA,IAAA,CAAK,mBAAmB,QAAQ,CAAA;AAAA,EAClC,CAAA;AAAA,EAEiB,eAAe,MAAY;AAC1C,IAAA,IAAA,CAAK,kBAAA,EAAmB;AAAA,EAC1B,CAAA;AAAA,EAEiB,gBAAgB,MAAY;AAC3C,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,mBAAmB,YAAY,CAAA;AAAA,EACtC,CAAA;AAAA,EAEiB,iBAAiB,MAAY;AAC5C,IAAA,IAAA,CAAK,kBAAA,EAAmB;AAAA,EAC1B,CAAA;AAAA,EAEiB,cAAA,GAAiB,CAAC,KAAA,KAAqC;AACtE,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAI,MAAM,SAAA,EAAW;AACnB,MAAA,IAAA,CAAK,mBAAmB,UAAU,CAAA;AAAA,IACpC;AAAA,EACF,CAAA;AAAA,EAEiB,qBAAqB,MAAY;AAChD,IAAA,KAAK,KAAK,sBAAA,EAAuB;AAAA,EACnC,CAAA;AAAA,EAEA,MAAc,sBAAA,GAAwC;AACpD,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,oBAAA,EAAsB;AACzC,MAAA;AAAA,IACF;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,EAAE,WAAA,EAAa,WAAA,EAAY,GAAI,MAAM,sBAAA,CAAuB;AAAA,QAChE,aAAa,IAAA,CAAK,WAAA;AAAA,QAClB,aAAa,IAAA,CAAK,WAAA;AAAA,QAClB,gBAAgB,IAAA,CAAK;AAAA,OACtB,CAAA;AACD,MAAA,IAAA,CAAK,WAAA,GAAc,WAAA;AACnB,MAAA,IAAA,CAAK,WAAA,GAAc,WAAA;AAAA,IACrB,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,OAAA;AAAA,QACN,KAAA,EAAO,iBAAiB,KAAA,GAAQ,KAAA,GAAQ,IAAI,KAAA,CAAM,MAAA,CAAO,KAAK,CAAC;AAAA,OAChE,CAAA;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,kBAAA,CACN,OACA,KAAA,EACM;AACN,IAAA,IACE,KAAK,sBAAA,IACL,IAAA,CAAK,oBAAA,IACL,IAAA,CAAK,gBACL,CAAC,IAAA,CAAK,EAAA,IACN,CAAC,KAAK,EAAA,IACN,CAAC,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA,EACzB;AACA,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,SAAA,CAAU;AAAA,MACb,IAAA,EAAM,iBAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,cAAc,KAAA,EAAwC;AAC5D,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,CAAC,KAAK,EAAA,EAAI;AACxB,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IACE,UAAU,YAAA,IACV,KAAA,KAAU,cACV,KAAA,KAAU,SAAA,IACV,UAAU,QAAA,EACV;AACA,MAAA,OACE,KAAK,EAAA,CAAG,UAAA,KAAe,UACvB,IAAA,CAAK,EAAA,CAAG,oBAAoB,QAAA,IAC5B,IAAA,CAAK,EAAA,CAAG,eAAA,KAAoB,YAC5B,IAAA,CAAK,EAAA,CAAG,uBAAuB,QAAA,IAC/B,IAAA,CAAK,GAAG,kBAAA,KAAuB,cAAA;AAAA,IAEnC;AAEA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GACJ,MAAA,CAAO,YAAA,IACN,MAAA,CACE,kBAAA;AACL,IAAA,IAAI,CAAC,uBAAA,EAAyB;AAC5B,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,YAAA,GAAe,IAAI,uBAAA,EAAwB;AAAA,EAClD;AAAA,EAEQ,mBAAmB,MAAA,EAA2B;AACpD,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAC1B,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,aAAa,UAAA,EAAW;AAC7B,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA,CAAK,YAAA,CAAa,uBAAA,CAAwB,MAAM,CAAA;AACnE,MAAA,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,YAAA,CAAa,cAAA,EAAe;AACjD,MAAA,IAAA,CAAK,SAAS,OAAA,GAAU,GAAA;AACxB,MAAA,IAAA,CAAK,SAAS,qBAAA,GAAwB,GAAA;AACtC,MAAA,IAAA,CAAK,WAAA,CAAY,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AAEtC,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,IAC5B,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,mCAAmC,KAAK,CAAA;AAAA,IACxD;AAAA,EACF;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,QAAA,CAAS,iBAAA;AACnC,IAAA,MAAM,SAAA,GAAY,IAAI,UAAA,CAAW,YAAY,CAAA;AAE7C,IAAA,IAAA,CAAK,eAAA,GAAkB,MAAA,CAAO,WAAA,CAAY,MAAM;AAC9C,MAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,QAAA,CAAS,qBAAqB,SAAS,CAAA;AAE5C,MAAA,IAAI,GAAA,GAAM,CAAA;AACV,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,YAAA,EAAc,SAAS,CAAA,EAAG;AACpD,QAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAK,CAAA,GAAI,GAAA;AACtC,QAAA,GAAA,IAAO,UAAA,GAAa,UAAA;AAAA,MACtB;AAEA,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,IAAA,CAAK,GAAA,GAAM,YAAY,CAAA;AACxC,MAAA,IAAA,CAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,eAAA;AAAA,QACN,GAAA,EAAK,IAAA,CAAK,GAAA,CAAI,GAAA,GAAM,GAAG,CAAC;AAAA,OACzB,CAAA;AAAA,IACH,CAAA,EAAG,MAAO,EAAE,CAAA;AAAA,EACd;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,aAAA,CAAc,KAAK,eAAe,CAAA;AAClC,MAAA,IAAA,CAAK,eAAA,GAAkB,IAAA;AAAA,IACzB;AAEA,IAAA,IAAA,CAAK,UAAU,EAAE,IAAA,EAAM,eAAA,EAAiB,GAAA,EAAK,GAAG,CAAA;AAAA,EAClD;AAAA,EAEQ,kBAAA,GAA2B;AACjC,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,CAAC,IAAA,CAAK,QAAA,IAAY,IAAA,CAAK,eAAA,EAAiB;AAC1C,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEQ,OAAA,GAAgB;AACtB,IAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AACpB,IAAA,IAAA,CAAK,8BAAA,EAA+B;AACpC,IAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,0BAAA;AAAA,MACH,IAAI,MAAM,wDAAwD;AAAA,KACpE;AAEA,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,WAAA,EAAa;AACpB,MAAA,IAAA,CAAK,WAAA,CAAY,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAC5D,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAAA,IACrB;AAEA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAA,CAAK,aAAa,SAAA,GAAY,IAAA;AAC9B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAA,CAAK,aAAa,KAAA,EAAM;AACxB,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AACnB,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AACnB,IAAA,IAAA,CAAK,QAAA,GAAW,IAAA;AAChB,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,IAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,IAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAAA,EAC7B;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AAAA,EACxB;AAAA,EAEQ,sBAAA,GAAyC;AAC/C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,OAAO,IAAA,CAAK,EAAA;AAAA,EACd;AAAA,EAEA,MAAc,sBACZ,MAAA,EACkB;AAClB,IAAA,MAAM,KAAK,IAAA,CAAK,EAAA;AAChB,IAAA,IAAI,CAAC,EAAA,IAAM,IAAA,CAAK,uBAAA,EAAwB,EAAG;AACzC,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IAAI,EAAA,CAAG,uBAAuB,WAAA,EAAa;AACzC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAI,EAAA,CAAG,uBAAuB,cAAA,EAAgB;AAC5C,MAAA,MAAM,MAAM,IAAK,CAAA;AACjB,MAAA,IAAI,EAAA,CAAG,uBAAuB,cAAA,EAAgB;AAC5C,QAAA,OAAO,IAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,IACE,EAAA,CAAG,uBAAuB,QAAA,IAC1B,EAAA,CAAG,oBAAoB,QAAA,IACvB,EAAA,CAAG,oBAAoB,QAAA,EACvB;AACA,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IAAI,OAAO,EAAA,CAAG,UAAA,KAAe,UAAA,EAAY;AACvC,MAAA,EAAA,CAAG,UAAA,EAAW;AAAA,IAChB;AAEA,IAAA,MAAM,QAAQ,MAAM,EAAA,CAAG,YAAY,EAAE,UAAA,EAAY,MAAM,CAAA;AACvD,IAAA,MAAM,EAAA,CAAG,oBAAoB,KAAK,CAAA;AAElC,IAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,mBAAA,CAAoB;AAAA,MAC/C,SAAA,EAAW,QAAA;AAAA,MACX,OAAA,EAAS,MAAA,IAAU,IAAA,CAAK,oBAAA,IAAwB,EAAC;AAAA,MACjD,UAAU,KAAA,CAAM;AAAA,KACjB,CAAA;AAED,IAAA,IACE,SAAA,CAAU,YAAY,uBAAA,IACtB,SAAA,CAAU,cAAc,QAAA,IACxB,EAAE,eAAe,SAAA,CAAA,EACjB;AACA,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,wCAAwC,uBAAuB,CAAA,qBAAA,EAAwB,UAAU,OAAO,CAAA,CAAA,EAAI,UAAU,SAAS,CAAA;AAAA,OACjI;AAAA,IACF;AAEA,IAAA,MAAM,GAAG,oBAAA,CAAqB;AAAA,MAC5B,IAAA,EAAM,QAAA;AAAA,MACN,KAAK,SAAA,CAAU;AAAA,KAChB,CAAA;AACD,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEQ,uBAAA,GAAmC;AACzC,IAAA,OACE,CAAC,KAAK,EAAA,IACN,CAAC,KAAK,EAAA,IACN,IAAA,CAAK,GAAG,UAAA,KAAe,QAAA,IACvB,KAAK,EAAA,CAAG,eAAA,KAAoB,YAC5B,IAAA,CAAK,EAAA,CAAG,oBAAoB,QAAA,IAC5B,IAAA,CAAK,GAAG,kBAAA,KAAuB,QAAA;AAAA,EAEnC;AAAA,EAEA,MAAc,kBACZ,MAAA,EACe;AACf,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,OAAA,EAAQ;AACb,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,IAAU,IAAA,CAAK,oBAAoB,CAAA;AAAA,EACxD;AAAA,EAEQ,2BAAA,GAAoC;AAC1C,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,YAAA,CAAa,IAAA,CAAK,qBAAqB,SAAS,CAAA;AAChD,IAAA,IAAA,CAAK,qBAAqB,OAAA,EAAQ;AAClC,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAAA,EAC9B;AAAA,EAEQ,2BAA2B,KAAA,EAAoB;AACrD,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,YAAA,CAAa,IAAA,CAAK,qBAAqB,SAAS,CAAA;AAChD,IAAA,IAAA,CAAK,oBAAA,CAAqB,OAAO,KAAK,CAAA;AACtC,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAAA,EAC9B;AAAA,EAEQ,4BAA4B,KAAA,EAA6B;AAC/D,IAAA,MAAM,gBAAgB,IAAA,CAAK,oBAAA;AAC3B,IAAA,IAAI,CAAC,aAAA,EAAe;AAClB,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA,CAAM,KAAA,EAAO,QAAA,KAAa,aAAA,CAAc,OAAA;AAAA,EACjD;AAAA,EAEQ,yBAAA,GAAoC;AAC1C,IAAA,OACE,IAAA,CAAK,QAAQ,sBAAA,IAA0B,iCAAA;AAAA,EAE3C;AAAA,EAEQ,qBAAA,GAA8B;AACpC,IAAA,IAAA,CAAK,aAAA,GAAgB,EAAA;AACrB,IAAA,IAAA,CAAK,2BAAA,GAA8B,KAAA;AACnC,IAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAAA,EAC/B;AAAA,EAEQ,UAAU,KAAA,EAAqC;AACrD,IAAA,KAAA,MAAW,QAAA,IAAY,KAAK,cAAA,EAAgB;AAC1C,MAAA,QAAA,CAAS,KAAK,CAAA;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,OAAO,IAAA,EAAuB;AACpC,IAAA,IAAI,IAAA,CAAK,QAAQ,KAAA,EAAO;AACtB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAG,IAAI,CAAA;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,oBAAoB,SAAA,EAA4B;AACtD,IAAA,IAAI,qBAAA,CAAsB,GAAA,CAAI,SAAS,CAAA,EAAG;AACxC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,SAAA,CAAU,WAAW,uBAAuB,CAAA;AAAA,EACrD;AACF;AAEO,SAAS,2BACd,OAAA,EACyB;AACzB,EAAA,OAAO,IAAI,qBAAqB,OAAO,CAAA;AACzC;AAEA,SAAS,8BACP,MAAA,EACyB;AACzB,EAAA,MAAM,OAAA,GAAmC;AAAA,IACvC,KAAA,EAAO;AAAA,MACL,MAAA,EAAQ;AAAA,QACN,KAAA,EAAO,QAAQ,KAAA,IAAS;AAAA;AAC1B,KACF;AAAA,IACA,WAAA,EAAa,QAAQ,UAAA,IAAc;AAAA,GACrC;AAEA,EAAA,IAAI,QAAQ,YAAA,EAAc;AACxB,IAAA,OAAA,CAAQ,eAAe,MAAA,CAAO,YAAA;AAAA,EAChC;AAEA,EAAA,IAAI,MAAA,EAAQ,gBAAgB,MAAA,EAAW;AACrC,IAAA,OAAA,CAAQ,cAAc,MAAA,CAAO,WAAA;AAAA,EAC/B;AAEA,EAAA,IAAI,MAAA,EAAQ,cAAc,MAAA,EAAW;AACnC,IAAA,OAAA,CAAQ,6BAA6B,MAAA,CAAO,SAAA;AAAA,EAC9C;AAEA,EAAA,IAAI,MAAA,EAAQ,OAAO,MAAA,EAAQ;AACzB,IAAA,OAAA,CAAQ,QAAQ,MAAA,CAAO,KAAA;AAAA,EACzB;AAEA,EAAA,OAAO,OAAA;AACT;AAEA,SAAS,MAAM,EAAA,EAA2B;AACxC,EAAA,OAAO,IAAI,OAAA,CAAQ,CAAC,OAAA,KAAY;AAC9B,IAAA,UAAA,CAAW,SAAS,EAAE,CAAA;AAAA,EACxB,CAAC,CAAA;AACH","file":"chunk-DHNRDHST.mjs","sourcesContent":["import type {\n RealtimeReconnectCause,\n RealtimeSessionBootstrap,\n RealtimeSessionConfig,\n RealtimeSessionRequest,\n} from \"@charivo/core\";\nimport { OPENAI_REALTIME_ADAPTER } from \"@charivo/core\";\nimport { DEFAULT_REALTIME_VOICE } from \"../instructions\";\nimport { acquireMicrophoneStream } from \"../internal/microphone\";\nimport {\n bindTransportLifecycle,\n createIceDisconnectDebouncer,\n DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS,\n replaceMicrophoneTrack,\n} from \"../internal/webrtc-lifecycle\";\nimport {\n DEFAULT_REQUEST_TIMEOUT_MS,\n fetchWithTimeout,\n isRealtimeSessionBootstrap,\n isRecord,\n} from \"../internal/shared\";\nimport type { RealtimeTransportClient, RealtimeTransportEvent } from \"../types\";\n\ninterface ServerError {\n code?: string;\n event_id?: string;\n message?: string;\n}\n\ninterface ServerEventItem {\n call_id?: string;\n name?: string;\n arguments?: string;\n}\n\ninterface ServerEvent {\n type: string;\n event_id?: string;\n delta?: string;\n text?: string;\n transcript?: string;\n call_id?: string;\n item_id?: string;\n name?: string;\n arguments?: string;\n item?: ServerEventItem;\n error?: ServerError;\n}\n\nexport interface OpenAIRealtimeClientOptions {\n apiEndpoint?: string;\n debug?: boolean;\n sessionUpdateTimeoutMs?: number;\n sessionBootstrap?: (\n request: RealtimeSessionRequest,\n ) => Promise<RealtimeSessionBootstrap>;\n}\n\ninterface PendingSessionUpdate {\n eventId: string;\n resolve: () => void;\n reject: (error: Error) => void;\n timeoutId: ReturnType<typeof setTimeout>;\n}\n\nconst DEBUG_EVENT_ALLOWLIST = new Set<string>([\n \"session.created\",\n \"session.updated\",\n \"response.created\",\n \"response.done\",\n \"response.function_call_arguments.done\",\n \"conversation.item.input_audio_transcription.completed\",\n \"error\",\n]);\nconst DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5_000;\n\n/**\n * OpenAI-specific realtime transport client.\n *\n * This package normalizes OpenAI Realtime WebRTC events into the transport\n * event contract defined by `@charivo/realtime`.\n */\nexport class OpenAIRealtimeClient implements RealtimeTransportClient {\n private pc: RTCPeerConnection | null = null;\n private dc: RTCDataChannel | null = null;\n private audioElement: HTMLAudioElement | null = null;\n private mediaStream: MediaStream | null = null;\n private audioSender: RTCRtpSender | null = null;\n private audioContext: AudioContext | null = null;\n private audioSource: MediaStreamAudioSourceNode | null = null;\n private analyser: AnalyserNode | null = null;\n private lipSyncInterval: number | null = null;\n private isResponseInProgress = false;\n private cancelInFlight = false;\n private hasStartedAssistantResponse = false;\n private hasStartedAudioOutput = false;\n private assistantText = \"\";\n private nextSessionUpdateId = 1;\n private pendingSessionUpdate: PendingSessionUpdate | null = null;\n private currentSessionConfig?: RealtimeSessionConfig;\n private connectionLossNotified = false;\n private isExplicitDisconnect = false;\n private isRecovering = false;\n private isCleaningUp = false;\n private teardownTransportLifecycle?: () => void;\n private readonly iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {\n this.emitConnectionLost(\"ice-disconnected\");\n }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);\n private eventCallbacks = new Set<(event: RealtimeTransportEvent) => void>();\n\n constructor(private options: OpenAIRealtimeClientOptions = {}) {}\n\n async connect(config?: RealtimeSessionConfig): Promise<void> {\n try {\n if (this.isExplicitDisconnect && this.isRecovering) {\n throw new Error(\"Realtime session disconnect requested\");\n }\n\n this.log(\"Starting OpenAI Realtime WebRTC connection\");\n\n this.isExplicitDisconnect = false;\n this.isRecovering = false;\n this.connectionLossNotified = false;\n this.currentSessionConfig = config;\n this.pc = new RTCPeerConnection();\n this.audioElement = document.createElement(\"audio\");\n this.audioElement.autoplay = true;\n this.audioElement.setAttribute(\"playsinline\", \"true\");\n this.prepareAudioAnalysis();\n this.bindConnectionEvents();\n this.bindTransportLifecycleEvents();\n\n this.pc.ontrack = (event) => {\n if (this.audioElement) {\n this.audioElement.srcObject = event.streams[0];\n }\n this.setupAudioAnalysis(event.streams[0]);\n };\n\n try {\n this.mediaStream = await acquireMicrophoneStream();\n const audioTrack = this.mediaStream.getTracks()[0];\n this.audioSender =\n this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;\n } catch {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n this.dc = this.pc.createDataChannel(\"oai-events\");\n this.dc.onmessage = (event) => {\n const payload = JSON.parse(event.data) as ServerEvent;\n this.handleServerEvent(payload);\n };\n this.dc.onerror = () => {\n this.emitEvent({\n type: \"error\",\n error: new Error(\"DataChannel error\"),\n });\n };\n\n const offer = await this.pc.createOffer();\n await this.pc.setLocalDescription(offer);\n\n const bootstrap = await this.getSessionBootstrap({\n transport: \"webrtc\",\n session: config ?? {},\n sdpOffer: offer.sdp,\n });\n\n if (\n bootstrap.adapter !== OPENAI_REALTIME_ADAPTER ||\n bootstrap.transport !== \"webrtc\" ||\n !(\"answerSdp\" in bootstrap)\n ) {\n throw new Error(\n `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`,\n );\n }\n\n await this.pc.setRemoteDescription({\n type: \"answer\",\n sdp: bootstrap.answerSdp,\n });\n\n this.emitEvent({ type: \"session.started\" });\n } catch (error) {\n this.cleanup();\n throw error;\n }\n }\n\n async disconnect(): Promise<void> {\n this.log(\"Disconnecting OpenAI Realtime WebRTC\");\n this.isExplicitDisconnect = true;\n this.isRecovering = false;\n this.cleanup();\n }\n\n async updateSession(config?: RealtimeSessionConfig): Promise<void> {\n if (this.isResponseInProgress) {\n throw new Error(\n \"Cannot update the realtime session while a response is in progress. Call interrupt() first.\",\n );\n }\n\n const dc = this.requireOpenDataChannel();\n const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;\n\n if (this.pendingSessionUpdate) {\n throw new Error(\"Realtime session update already in progress\");\n }\n\n dc.send(\n JSON.stringify({\n type: \"session.update\",\n event_id: eventId,\n session: toOpenAIRealtimeSessionUpdate(config),\n }),\n );\n\n await new Promise<void>((resolve, reject) => {\n const timeoutId = setTimeout(() => {\n this.rejectPendingSessionUpdate(\n new Error(\n `Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`,\n ),\n );\n }, this.getSessionUpdateTimeoutMs());\n\n this.pendingSessionUpdate = {\n eventId,\n resolve,\n reject,\n timeoutId,\n };\n });\n\n this.currentSessionConfig = config;\n }\n\n async recover(config?: RealtimeSessionConfig): Promise<void> {\n this.isRecovering = true;\n this.connectionLossNotified = true;\n this.currentSessionConfig = config;\n\n try {\n if (await this.tryIceRestartRecovery(config)) {\n this.connectionLossNotified = false;\n return;\n }\n\n await this.rebuildConnection(config);\n this.connectionLossNotified = false;\n } finally {\n this.isRecovering = false;\n }\n }\n\n async sendText(text: string): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n if (this.isResponseInProgress) {\n console.warn(\"⚠️ Response already in progress, skipping request\");\n return;\n }\n\n this.dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text,\n },\n ],\n },\n }),\n );\n this.dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async sendAudio(_audio: ArrayBuffer): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n console.warn(\n \"sendAudio is not needed with WebRTC - audio is automatically transmitted\",\n );\n }\n\n async sendToolResult(\n callId: string,\n output: Record<string, unknown>,\n ): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n const dc = this.requireOpenDataChannel();\n\n dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"function_call_output\",\n call_id: callId,\n output: JSON.stringify(output),\n },\n }),\n );\n dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async interrupt(): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n const dc = this.requireOpenDataChannel();\n\n if (!this.isResponseInProgress) {\n return;\n }\n\n dc.send(JSON.stringify({ type: \"response.cancel\" }));\n this.isResponseInProgress = false;\n this.cancelInFlight = true;\n }\n\n onEvent(callback: (event: RealtimeTransportEvent) => void): void {\n this.eventCallbacks.add(callback);\n }\n\n private handleServerEvent(event: ServerEvent): void {\n if (this.options.debug && this.shouldLogDebugEvent(event.type)) {\n this.log(\"📡 [OpenAI Realtime Event]\", event.type, event);\n }\n\n switch (event.type) {\n case \"session.created\":\n return;\n\n case \"session.updated\":\n this.resolvePendingSessionUpdate();\n return;\n\n case \"response.audio.delta\":\n case \"output_audio_buffer.started\":\n if (this.cancelInFlight) {\n return;\n }\n\n if (!this.hasStartedAudioOutput) {\n this.hasStartedAudioOutput = true;\n this.emitEvent({ type: \"audio.output.started\" });\n }\n return;\n\n case \"response.audio.done\":\n case \"response.output_audio.done\":\n case \"output_audio_buffer.stopped\":\n if (this.cancelInFlight) {\n this.hasStartedAudioOutput = false;\n return;\n }\n\n this.emitEvent({ type: \"audio.output.ended\" });\n this.hasStartedAudioOutput = false;\n return;\n\n case \"response.done\":\n this.isResponseInProgress = false;\n if (this.cancelInFlight) {\n this.cancelInFlight = false;\n this.resetResponseTracking();\n return;\n }\n\n // Tool-using turns fire response.done twice per user message: once\n // after the tool call (no text this cycle) and once after the\n // follow-up reply. Skip the first one so consumers see a single\n // completion per user turn. Keeping tracking live means the\n // follow-up cycle does not re-emit assistant.response.started.\n if (!this.assistantText) {\n return;\n }\n\n this.emitEvent({\n type: \"assistant.response.completed\",\n text: this.assistantText,\n });\n this.resetResponseTracking();\n return;\n\n case \"response.audio_transcript.delta\":\n case \"response.output_audio_transcript.delta\":\n case \"response.output_text.delta\":\n if (this.cancelInFlight || !event.delta) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n this.assistantText += event.delta;\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: event.delta,\n });\n return;\n\n case \"response.output_text.done\":\n if (this.cancelInFlight || !event.text) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n if (event.text !== this.assistantText) {\n const delta = event.text.startsWith(this.assistantText)\n ? event.text.slice(this.assistantText.length)\n : event.text;\n\n this.assistantText = event.text;\n if (delta) {\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: delta,\n });\n }\n }\n return;\n\n case \"response.output_audio_transcript.done\":\n if (this.cancelInFlight || !event.transcript) {\n return;\n }\n\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n\n if (event.transcript !== this.assistantText) {\n const delta = event.transcript.startsWith(this.assistantText)\n ? event.transcript.slice(this.assistantText.length)\n : event.transcript;\n\n this.assistantText = event.transcript;\n if (delta) {\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: delta,\n });\n }\n }\n return;\n\n case \"conversation.item.input_audio_transcription.completed\":\n if (event.transcript) {\n this.emitEvent({\n type: \"user.transcript\",\n text: event.transcript,\n });\n }\n return;\n\n case \"response.function_call_arguments.done\":\n this.handleToolCallDone(event);\n return;\n\n case \"error\":\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n {\n const error = new Error(event.error?.message || \"Unknown error\");\n if (this.isPendingSessionUpdateError(event)) {\n this.rejectPendingSessionUpdate(error);\n }\n\n this.emitEvent({\n type: \"error\",\n error,\n });\n }\n return;\n\n default:\n return;\n }\n }\n\n private handleToolCallDone(event: ServerEvent): void {\n const callId = event.call_id || event.item?.call_id || event.item_id;\n const name = event.name || event.item?.name;\n const argsJson = event.arguments || event.item?.arguments;\n\n if (!name || !argsJson) {\n console.warn(\"⚠️ Tool call done but missing name or arguments\");\n return;\n }\n\n let args: Record<string, unknown> = {};\n try {\n const parsed = JSON.parse(argsJson) as unknown;\n if (isRecord(parsed)) {\n args = parsed;\n }\n } catch (error) {\n console.error(\"Failed to parse tool call args\", error, argsJson);\n return;\n }\n\n this.emitEvent({\n type: \"tool.call\",\n name,\n args,\n callId,\n });\n }\n\n private async getSessionBootstrap(\n request: RealtimeSessionRequest,\n ): Promise<RealtimeSessionBootstrap> {\n if (this.options.sessionBootstrap) {\n return this.options.sessionBootstrap(request);\n }\n\n const apiEndpoint = this.options.apiEndpoint;\n if (!apiEndpoint) {\n throw new Error(\n \"OpenAI realtime client requires apiEndpoint or sessionBootstrap\",\n );\n }\n\n const response = await fetchWithTimeout(\n apiEndpoint,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(request),\n },\n `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,\n );\n\n if (!response.ok) {\n const errorText = await response.text();\n throw new Error(`Failed to create Realtime session: ${errorText}`);\n }\n\n const bootstrap = (await response.json()) as unknown;\n if (!isRealtimeSessionBootstrap(bootstrap)) {\n throw new Error(\"Invalid realtime session bootstrap response\");\n }\n\n return bootstrap as RealtimeSessionBootstrap;\n }\n\n private bindConnectionEvents(): void {\n if (!this.pc) {\n return;\n }\n\n this.pc.addEventListener(\"iceconnectionstatechange\", () => {\n const iceState = this.pc?.iceConnectionState;\n if (iceState === \"failed\") {\n this.iceDisconnectDebouncer.cancel();\n this.emitConnectionLost(\"ice-failed\");\n } else if (iceState === \"disconnected\") {\n this.iceDisconnectDebouncer.schedule();\n } else {\n this.iceDisconnectDebouncer.cancel();\n }\n });\n\n this.pc.addEventListener(\"connectionstatechange\", () => {\n const connectionState = this.pc?.connectionState;\n if (connectionState === \"failed\" || connectionState === \"closed\") {\n this.iceDisconnectDebouncer.cancel();\n this.emitConnectionLost(\"connection-failed\");\n }\n });\n }\n\n private bindTransportLifecycleEvents(): void {\n if (this.teardownTransportLifecycle) {\n return;\n }\n\n this.teardownTransportLifecycle = bindTransportLifecycle({\n onHidden: this.handleHidden,\n onOnline: this.handleOnline,\n onPageHide: this.handlePageHide,\n onPageShow: this.handlePageShow,\n onVisible: this.handleVisible,\n onDeviceChange: this.handleDeviceChange,\n });\n }\n\n private unbindTransportLifecycleEvents(): void {\n this.teardownTransportLifecycle?.();\n this.teardownTransportLifecycle = undefined;\n }\n\n private readonly handleOnline = (): void => {\n this.emitConnectionLost(\"online\");\n };\n\n private readonly handleHidden = (): void => {\n this.pauseAudioAnalysis();\n };\n\n private readonly handleVisible = (): void => {\n this.resumeAudioAnalysis();\n this.emitConnectionLost(\"visibility\");\n };\n\n private readonly handlePageHide = (): void => {\n this.pauseAudioAnalysis();\n };\n\n private readonly handlePageShow = (event: PageTransitionEvent): void => {\n this.resumeAudioAnalysis();\n if (event.persisted) {\n this.emitConnectionLost(\"pageshow\");\n }\n };\n\n private readonly handleDeviceChange = (): void => {\n void this.refreshMicrophoneTrack();\n };\n\n private async refreshMicrophoneTrack(): Promise<void> {\n if (!this.pc || this.isExplicitDisconnect) {\n return;\n }\n\n try {\n const { audioSender, mediaStream } = await replaceMicrophoneTrack({\n audioSender: this.audioSender,\n mediaStream: this.mediaStream,\n peerConnection: this.pc,\n });\n this.audioSender = audioSender;\n this.mediaStream = mediaStream;\n } catch (error) {\n this.emitEvent({\n type: \"error\",\n error: error instanceof Error ? error : new Error(String(error)),\n });\n }\n }\n\n private emitConnectionLost(\n cause: RealtimeReconnectCause,\n error?: Error,\n ): void {\n if (\n this.connectionLossNotified ||\n this.isExplicitDisconnect ||\n this.isCleaningUp ||\n !this.pc ||\n !this.dc ||\n !this.needsRecovery(cause)\n ) {\n return;\n }\n\n this.connectionLossNotified = true;\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.resetResponseTracking();\n this.stopLipSyncAnalysis();\n this.emitEvent({\n type: \"connection.lost\",\n cause,\n error,\n });\n }\n\n private needsRecovery(cause: RealtimeReconnectCause): boolean {\n if (!this.pc || !this.dc) {\n return false;\n }\n\n if (\n cause === \"visibility\" ||\n cause === \"pageshow\" ||\n cause === \"offline\" ||\n cause === \"online\"\n ) {\n return (\n this.dc.readyState !== \"open\" ||\n this.pc.connectionState === \"failed\" ||\n this.pc.connectionState === \"closed\" ||\n this.pc.iceConnectionState === \"failed\" ||\n this.pc.iceConnectionState === \"disconnected\"\n );\n }\n\n return true;\n }\n\n private prepareAudioAnalysis(): void {\n if (this.audioContext) {\n return;\n }\n\n const audioContextConstructor =\n window.AudioContext ||\n (window as Window & { webkitAudioContext?: typeof AudioContext })\n .webkitAudioContext;\n if (!audioContextConstructor) {\n return;\n }\n\n this.audioContext = new audioContextConstructor();\n }\n\n private setupAudioAnalysis(stream: MediaStream): void {\n try {\n this.prepareAudioAnalysis();\n if (!this.audioContext) {\n return;\n }\n\n this.audioSource?.disconnect();\n this.audioSource = this.audioContext.createMediaStreamSource(stream);\n this.analyser = this.audioContext.createAnalyser();\n this.analyser.fftSize = 256;\n this.analyser.smoothingTimeConstant = 0.8;\n this.audioSource.connect(this.analyser);\n\n this.startLipSyncAnalysis();\n } catch (error) {\n console.error(\"Failed to setup audio analysis:\", error);\n }\n }\n\n private startLipSyncAnalysis(): void {\n if (!this.analyser) {\n return;\n }\n\n const bufferLength = this.analyser.frequencyBinCount;\n const dataArray = new Uint8Array(bufferLength);\n\n this.lipSyncInterval = window.setInterval(() => {\n if (!this.analyser) {\n return;\n }\n\n this.analyser.getByteFrequencyData(dataArray);\n\n let sum = 0;\n for (let index = 0; index < bufferLength; index += 1) {\n const normalized = dataArray[index] / 255;\n sum += normalized * normalized;\n }\n\n const rms = Math.sqrt(sum / bufferLength);\n this.emitEvent({\n type: \"audio.lipsync\",\n rms: Math.min(rms * 3, 1),\n });\n }, 1000 / 60);\n }\n\n private stopLipSyncAnalysis(): void {\n if (this.lipSyncInterval) {\n clearInterval(this.lipSyncInterval);\n this.lipSyncInterval = null;\n }\n\n this.emitEvent({ type: \"audio.lipsync\", rms: 0 });\n }\n\n private pauseAudioAnalysis(): void {\n this.stopLipSyncAnalysis();\n }\n\n private resumeAudioAnalysis(): void {\n if (!this.analyser || this.lipSyncInterval) {\n return;\n }\n\n this.startLipSyncAnalysis();\n }\n\n private cleanup(): void {\n this.isCleaningUp = true;\n this.unbindTransportLifecycleEvents();\n this.iceDisconnectDebouncer.cancel();\n this.stopLipSyncAnalysis();\n this.rejectPendingSessionUpdate(\n new Error(\"Realtime session ended before session update completed\"),\n );\n\n if (this.dc) {\n this.dc.close();\n this.dc = null;\n }\n\n if (this.pc) {\n this.pc.close();\n this.pc = null;\n }\n\n if (this.mediaStream) {\n this.mediaStream.getTracks().forEach((track) => track.stop());\n this.mediaStream = null;\n }\n\n if (this.audioElement) {\n this.audioElement.srcObject = null;\n this.audioElement = null;\n }\n\n if (this.audioContext) {\n this.audioContext.close();\n this.audioContext = null;\n }\n\n this.audioSource = null;\n this.audioSender = null;\n this.analyser = null;\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.connectionLossNotified = false;\n this.isCleaningUp = false;\n this.resetResponseTracking();\n }\n\n private beginResponseRequest(): void {\n this.resetResponseTracking();\n this.isResponseInProgress = true;\n this.cancelInFlight = false;\n }\n\n private requireOpenDataChannel(): RTCDataChannel {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n return this.dc;\n }\n\n private async tryIceRestartRecovery(\n config?: RealtimeSessionConfig,\n ): Promise<boolean> {\n const pc = this.pc;\n if (!pc || this.shouldRebuildConnection()) {\n return false;\n }\n\n if (pc.iceConnectionState === \"connected\") {\n return true;\n }\n\n if (pc.iceConnectionState === \"disconnected\") {\n await delay(1_500);\n if (pc.iceConnectionState !== \"disconnected\") {\n return true;\n }\n }\n\n if (\n pc.iceConnectionState === \"failed\" ||\n pc.connectionState === \"failed\" ||\n pc.connectionState === \"closed\"\n ) {\n return false;\n }\n\n if (typeof pc.restartIce === \"function\") {\n pc.restartIce();\n }\n\n const offer = await pc.createOffer({ iceRestart: true });\n await pc.setLocalDescription(offer);\n\n const bootstrap = await this.getSessionBootstrap({\n transport: \"webrtc\",\n session: config ?? this.currentSessionConfig ?? {},\n sdpOffer: offer.sdp,\n });\n\n if (\n bootstrap.adapter !== OPENAI_REALTIME_ADAPTER ||\n bootstrap.transport !== \"webrtc\" ||\n !(\"answerSdp\" in bootstrap)\n ) {\n throw new Error(\n `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`,\n );\n }\n\n await pc.setRemoteDescription({\n type: \"answer\",\n sdp: bootstrap.answerSdp,\n });\n return true;\n }\n\n private shouldRebuildConnection(): boolean {\n return (\n !this.pc ||\n !this.dc ||\n this.dc.readyState === \"closed\" ||\n this.pc.connectionState === \"failed\" ||\n this.pc.connectionState === \"closed\" ||\n this.pc.iceConnectionState === \"failed\"\n );\n }\n\n private async rebuildConnection(\n config?: RealtimeSessionConfig,\n ): Promise<void> {\n this.connectionLossNotified = true;\n this.cleanup();\n await this.connect(config ?? this.currentSessionConfig);\n }\n\n private resolvePendingSessionUpdate(): void {\n if (!this.pendingSessionUpdate) {\n return;\n }\n\n clearTimeout(this.pendingSessionUpdate.timeoutId);\n this.pendingSessionUpdate.resolve();\n this.pendingSessionUpdate = null;\n }\n\n private rejectPendingSessionUpdate(error: Error): void {\n if (!this.pendingSessionUpdate) {\n return;\n }\n\n clearTimeout(this.pendingSessionUpdate.timeoutId);\n this.pendingSessionUpdate.reject(error);\n this.pendingSessionUpdate = null;\n }\n\n private isPendingSessionUpdateError(event: ServerEvent): boolean {\n const pendingUpdate = this.pendingSessionUpdate;\n if (!pendingUpdate) {\n return false;\n }\n\n return event.error?.event_id === pendingUpdate.eventId;\n }\n\n private getSessionUpdateTimeoutMs(): number {\n return (\n this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS\n );\n }\n\n private resetResponseTracking(): void {\n this.assistantText = \"\";\n this.hasStartedAssistantResponse = false;\n this.hasStartedAudioOutput = false;\n }\n\n private emitEvent(event: RealtimeTransportEvent): void {\n for (const callback of this.eventCallbacks) {\n callback(event);\n }\n }\n\n private log(...args: unknown[]): void {\n if (this.options.debug) {\n console.log(...args);\n }\n }\n\n private shouldLogDebugEvent(eventType: string): boolean {\n if (DEBUG_EVENT_ALLOWLIST.has(eventType)) {\n return true;\n }\n\n return eventType.startsWith(\"response.output_text.\");\n }\n}\n\nexport function createOpenAIRealtimeClient(\n options?: OpenAIRealtimeClientOptions,\n): RealtimeTransportClient {\n return new OpenAIRealtimeClient(options);\n}\n\nfunction toOpenAIRealtimeSessionUpdate(\n config?: RealtimeSessionConfig,\n): Record<string, unknown> {\n const session: Record<string, unknown> = {\n audio: {\n output: {\n voice: config?.voice ?? DEFAULT_REALTIME_VOICE,\n },\n },\n tool_choice: config?.toolChoice ?? \"auto\",\n };\n\n if (config?.instructions) {\n session.instructions = config.instructions;\n }\n\n if (config?.temperature !== undefined) {\n session.temperature = config.temperature;\n }\n\n if (config?.maxTokens !== undefined) {\n session.max_response_output_tokens = config.maxTokens;\n }\n\n if (config?.tools?.length) {\n session.tools = config.tools;\n }\n\n return session;\n}\n\nfunction delay(ms: number): Promise<void> {\n return new Promise((resolve) => {\n setTimeout(resolve, ms);\n });\n}\n"]}
|