@charivo/realtime 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-3PIMXNQU.mjs +222 -0
- package/dist/chunk-3PIMXNQU.mjs.map +1 -0
- package/dist/{chunk-QD4QSGPO.mjs → chunk-MI4QTRWT.mjs} +50 -136
- package/dist/chunk-MI4QTRWT.mjs.map +1 -0
- package/dist/{chunk-VYMT3Z2W.mjs → chunk-VSKMCGWD.mjs} +3 -136
- package/dist/chunk-VSKMCGWD.mjs.map +1 -0
- package/dist/chunk-XPKFJWHL.mjs +10 -0
- package/dist/chunk-XPKFJWHL.mjs.map +1 -0
- package/dist/index.mjs +1 -7
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +4 -10
- package/dist/openai/index.d.ts +4 -10
- package/dist/openai/index.js +186 -133
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.mjs +2 -2
- package/dist/remote/index.js +55 -133
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -3
- package/dist/remote/index.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-QD4QSGPO.mjs.map +0 -1
- package/dist/chunk-VYMT3Z2W.mjs.map +0 -1
- package/dist/chunk-YQWOBEE5.mjs +0 -89
- package/dist/chunk-YQWOBEE5.mjs.map +0 -1
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS } from './chunk-PNBYFAER.mjs';
|
|
2
|
+
import { subscribeBrowserLifecycle } from '@charivo/core';
|
|
3
|
+
|
|
4
|
+
// src/internal/microphone.ts
|
|
5
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
6
|
+
echoCancellation: true,
|
|
7
|
+
noiseSuppression: true,
|
|
8
|
+
autoGainControl: true
|
|
9
|
+
};
|
|
10
|
+
async function acquireMicrophoneStream() {
|
|
11
|
+
return navigator.mediaDevices.getUserMedia({
|
|
12
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
16
|
+
function bindTransportLifecycle(callbacks) {
|
|
17
|
+
const teardownBrowserLifecycle = subscribeBrowserLifecycle({
|
|
18
|
+
onHidden: callbacks.onHidden,
|
|
19
|
+
onOnline: callbacks.onOnline,
|
|
20
|
+
onPageHide: callbacks.onPageHide,
|
|
21
|
+
onPageShow: callbacks.onPageShow,
|
|
22
|
+
onVisible: callbacks.onVisible
|
|
23
|
+
});
|
|
24
|
+
if (callbacks.onDeviceChange) {
|
|
25
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
26
|
+
"devicechange",
|
|
27
|
+
callbacks.onDeviceChange
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
return () => {
|
|
31
|
+
teardownBrowserLifecycle();
|
|
32
|
+
if (callbacks.onDeviceChange) {
|
|
33
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
34
|
+
"devicechange",
|
|
35
|
+
callbacks.onDeviceChange
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
async function replaceMicrophoneTrack(state) {
|
|
41
|
+
if (!state.peerConnection) {
|
|
42
|
+
throw new Error("Realtime transport not active");
|
|
43
|
+
}
|
|
44
|
+
const nextStream = await acquireMicrophoneStream();
|
|
45
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
46
|
+
if (!nextTrack) {
|
|
47
|
+
throw new Error("Microphone access required for Realtime API");
|
|
48
|
+
}
|
|
49
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
50
|
+
if (!sender) {
|
|
51
|
+
throw new Error("No outbound audio sender available");
|
|
52
|
+
}
|
|
53
|
+
await sender.replaceTrack(nextTrack);
|
|
54
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
55
|
+
return {
|
|
56
|
+
audioSender: sender,
|
|
57
|
+
mediaStream: nextStream
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
61
|
+
let pending = null;
|
|
62
|
+
return {
|
|
63
|
+
schedule() {
|
|
64
|
+
if (pending) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
pending = setTimeout(() => {
|
|
68
|
+
pending = null;
|
|
69
|
+
onDisconnect();
|
|
70
|
+
}, delayMs);
|
|
71
|
+
},
|
|
72
|
+
cancel() {
|
|
73
|
+
if (!pending) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
clearTimeout(pending);
|
|
77
|
+
pending = null;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// src/openai-agents/lip-sync-analyzer.ts
|
|
83
|
+
var LipSyncAnalyzer = class {
|
|
84
|
+
constructor(options) {
|
|
85
|
+
this.options = options;
|
|
86
|
+
}
|
|
87
|
+
audioAnalysisStream = null;
|
|
88
|
+
audioContext = null;
|
|
89
|
+
audioSource = null;
|
|
90
|
+
analyser = null;
|
|
91
|
+
lipSyncInterval = null;
|
|
92
|
+
pendingAudioElementPoll = null;
|
|
93
|
+
observedAudioElement = null;
|
|
94
|
+
observedAudioElementListener = null;
|
|
95
|
+
async prepareAudioContext() {
|
|
96
|
+
if (this.audioContext) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
100
|
+
if (!audioContextConstructor) {
|
|
101
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
102
|
+
}
|
|
103
|
+
this.audioContext = new audioContextConstructor();
|
|
104
|
+
}
|
|
105
|
+
observeAudioElement(audioElement) {
|
|
106
|
+
this.detachObservedAudioElement();
|
|
107
|
+
const tryAttachStream = () => {
|
|
108
|
+
const stream = audioElement.srcObject;
|
|
109
|
+
if (stream instanceof MediaStream) {
|
|
110
|
+
this.attachStream(stream);
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
this.observedAudioElement = audioElement;
|
|
114
|
+
this.observedAudioElementListener = tryAttachStream;
|
|
115
|
+
audioElement.addEventListener("loadedmetadata", tryAttachStream);
|
|
116
|
+
this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
|
|
117
|
+
}
|
|
118
|
+
attachStream(stream) {
|
|
119
|
+
if (this.audioAnalysisStream === stream) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
this.stopAudioElementPolling();
|
|
123
|
+
this.stopLipSyncAnalysis();
|
|
124
|
+
this.audioAnalysisStream = stream;
|
|
125
|
+
try {
|
|
126
|
+
if (!this.audioContext) {
|
|
127
|
+
void this.prepareAudioContext().catch((error) => {
|
|
128
|
+
this.options.onError?.(error);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
if (!this.audioContext) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
this.audioSource?.disconnect?.();
|
|
135
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
136
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
137
|
+
this.analyser.fftSize = 256;
|
|
138
|
+
this.analyser.smoothingTimeConstant = 0.8;
|
|
139
|
+
this.audioSource.connect(this.analyser);
|
|
140
|
+
this.startLipSyncAnalysis();
|
|
141
|
+
} catch (error) {
|
|
142
|
+
this.options.onError?.(error);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
stopOutput() {
|
|
146
|
+
this.stopLipSyncAnalysis();
|
|
147
|
+
}
|
|
148
|
+
pause() {
|
|
149
|
+
this.stopLipSyncAnalysis();
|
|
150
|
+
}
|
|
151
|
+
resume() {
|
|
152
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
this.startLipSyncAnalysis();
|
|
156
|
+
}
|
|
157
|
+
cleanup() {
|
|
158
|
+
this.stopLipSyncAnalysis();
|
|
159
|
+
this.stopAudioElementPolling();
|
|
160
|
+
this.detachObservedAudioElement();
|
|
161
|
+
this.audioSource?.disconnect?.();
|
|
162
|
+
this.audioSource = null;
|
|
163
|
+
if (this.audioContext) {
|
|
164
|
+
void this.audioContext.close();
|
|
165
|
+
this.audioContext = null;
|
|
166
|
+
}
|
|
167
|
+
this.audioAnalysisStream = null;
|
|
168
|
+
this.analyser = null;
|
|
169
|
+
}
|
|
170
|
+
startLipSyncAnalysis() {
|
|
171
|
+
if (!this.analyser) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
175
|
+
const dataArray = new Uint8Array(bufferLength);
|
|
176
|
+
this.lipSyncInterval = window.setInterval(() => {
|
|
177
|
+
if (!this.analyser) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
this.analyser.getByteFrequencyData(dataArray);
|
|
181
|
+
let sum = 0;
|
|
182
|
+
for (let index = 0; index < bufferLength; index += 1) {
|
|
183
|
+
const normalized = dataArray[index] / 255;
|
|
184
|
+
sum += normalized * normalized;
|
|
185
|
+
}
|
|
186
|
+
const rms = Math.sqrt(sum / bufferLength);
|
|
187
|
+
this.options.onRms(Math.min(rms * 3, 1));
|
|
188
|
+
}, 1e3 / 60);
|
|
189
|
+
}
|
|
190
|
+
stopLipSyncAnalysis() {
|
|
191
|
+
if (this.lipSyncInterval) {
|
|
192
|
+
clearInterval(this.lipSyncInterval);
|
|
193
|
+
this.lipSyncInterval = null;
|
|
194
|
+
}
|
|
195
|
+
this.options.onRms(0);
|
|
196
|
+
}
|
|
197
|
+
stopAudioElementPolling() {
|
|
198
|
+
if (this.pendingAudioElementPoll) {
|
|
199
|
+
clearInterval(this.pendingAudioElementPoll);
|
|
200
|
+
this.pendingAudioElementPoll = null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
detachObservedAudioElement() {
|
|
204
|
+
if (this.observedAudioElement && this.observedAudioElementListener) {
|
|
205
|
+
this.observedAudioElement.removeEventListener(
|
|
206
|
+
"loadedmetadata",
|
|
207
|
+
this.observedAudioElementListener
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
this.observedAudioElement = null;
|
|
211
|
+
this.observedAudioElementListener = null;
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
// src/openai/defaults.ts
|
|
216
|
+
var DEFAULT_OPENAI_REALTIME_MODEL = "gpt-realtime-mini";
|
|
217
|
+
var DEFAULT_OPENAI_REALTIME_VOICE = "marin";
|
|
218
|
+
var DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
219
|
+
|
|
220
|
+
export { DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS, DEFAULT_OPENAI_REALTIME_MODEL, DEFAULT_OPENAI_REALTIME_VOICE, LipSyncAnalyzer, acquireMicrophoneStream, bindTransportLifecycle, createIceDisconnectDebouncer, replaceMicrophoneTrack };
|
|
221
|
+
//# sourceMappingURL=chunk-3PIMXNQU.mjs.map
|
|
222
|
+
//# sourceMappingURL=chunk-3PIMXNQU.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/internal/microphone.ts","../src/internal/webrtc-lifecycle.ts","../src/openai-agents/lip-sync-analyzer.ts","../src/openai/defaults.ts"],"names":[],"mappings":";;;;AAAO,IAAM,8BAAA,GAAiC;AAAA,EAC5C,gBAAA,EAAkB,IAAA;AAAA,EAClB,gBAAA,EAAkB,IAAA;AAAA,EAClB,eAAA,EAAiB;AACnB,CAAA;AAEA,eAAsB,uBAAA,GAAgD;AACpE,EAAA,OAAO,SAAA,CAAU,aAAa,YAAA,CAAa;AAAA,IACzC,KAAA,EAAO;AAAA,GACR,CAAA;AACH;ACPO,IAAM,oCAAA,GAAuC;AAW7C,SAAS,uBACd,SAAA,EACY;AACZ,EAAA,MAAM,2BAA2B,yBAAA,CAA0B;AAAA,IACzD,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,WAAW,SAAA,CAAU;AAAA,GACtB,CAAA;AAED,EAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,IAAA,SAAA,CAAU,YAAA,EAAc,gBAAA;AAAA,MACtB,cAAA;AAAA,MACA,SAAA,CAAU;AAAA,KACZ;AAAA,EACF;AAEA,EAAA,OAAO,MAAM;AACX,IAAA,wBAAA,EAAyB;AACzB,IAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,MAAA,SAAA,CAAU,YAAA,EAAc,mBAAA;AAAA,QACtB,cAAA;AAAA,QACA,SAAA,CAAU;AAAA,OACZ;AAAA,IACF;AAAA,EACF,CAAA;AACF;AAQA,eAAsB,uBACpB,KAAA,EACkE;AAClE,EAAA,IAAI,CAAC,MAAM,cAAA,EAAgB;AACzB,IAAA,MAAM,IAAI,MAAM,+BAA+B,CAAA;AAAA,EACjD;AAEA,EAAA,MAAM,UAAA,GAAa,MAAM,uBAAA,EAAwB;AACjD,EAAA,MAAM,SAAA,GAAY,UAAA,CAAW,cAAA,EAAe,CAAE,CAAC,CAAA;AAC/C,EAAA,IAAI,CAAC,SAAA,EAAW;AACd,IAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,EAC/D;AAEA,EAAA,MAAM,MAAA,GACJ,MAAM,WAAA,KACL,OAAO,MAAM,cAAA,CAAe,UAAA,KAAe,aACvC,KAAA,CAAM,cAAA,CACJ,YAAW,CACX,IAAA,CAAK,CAAC,SAAA,KAAc,SAAA,CAAU,OAAO,IAAA,KAAS,OAAO,KAAK,IAAA,GAC7D,IAAA,CAAA;AACN,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,EACtD;AAEA,EAAA,MAAM,MAAA,CAAO,aAAa,SAAS,CAAA;AACnC,EAAA,KAAA,CAAM,WAAA,EAAa,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAE9D,EAAA,OAAO;AAAA,IACL,WAAA,EAAa,MAAA;AAAA,IACb,WAAA,EAAa;AAAA,GACf;AACF;AAEO,SAAS,4BAAA,CACd,YAAA,EACA,OAAA,GAAU,oCAAA,EAIV;AACA,EAAA,IAAI,OAAA,GAAgD,IAAA;AAEpD,EAAA,OAAO;AAAA,IACL,QAAA,GAAW;AACT,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AAEA,MAAA,OAAA,GAAU,WAAW,MAAM;AACzB,QAAA,OAAA,GAAU,IAAA;AACV,QAAA,YAAA,EAAa;AAAA,MACf,GAAG,OAAO,CAAA;AAAA,IACZ,CAAA;AAAA,IACA,MAAA,GAAS;AACP,MAAA,IAAI,CAAC,OAAA,EAAS;AACZ,QAAA;AAAA,MACF;AAEA,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,OAAA,GAAU,IAAA;AAAA,IACZ;AAAA,GACF;AACF;;;AC1GO,IAAM,kBAAN,MAAsB;AAAA,EAU3B,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAAkC;AAAA,EAT9C,mBAAA,GAA0C,IAAA;AAAA,EAC1C,YAAA,GAAoC,IAAA;AAAA,EACpC,WAAA,GAAiD,IAAA;AAAA,EACjD,QAAA,GAAgC,IAAA;AAAA,EAChC,eAAA,GAAiC,IAAA;AAAA,EACjC,uBAAA,GAAyC,IAAA;AAAA,EACzC,oBAAA,GAAgD,IAAA;AAAA,EAChD,4BAAA,GAAoD,IAAA;AAAA,EAI5D,MAAM,mBAAA,GAAqC;AACzC,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GACJ,MAAA,CAAO,YAAA,IACN,MAAA,CACE,kBAAA;AACL,IAAA,IAAI,CAAC,uBAAA,EAAyB;AAC5B,MAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AAAA,IACjE;AAEA,IAAA,IAAA,CAAK,YAAA,GAAe,IAAI,uBAAA,EAAwB;AAAA,EAClD;AAAA,EAEA,oBAAoB,YAAA,EAAsC;AACxD,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,MAAM,kBAAkB,MAAM;AAC5B,MAAA,MAAM,SAAS,YAAA,CAAa,SAAA;AAC5B,MAAA,IAAI,kBAAkB,WAAA,EAAa;AACjC,QAAA,IAAA,CAAK,aAAa,MAAM,CAAA;AAAA,MAC1B;AAAA,IACF,CAAA;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,YAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,eAAA;AACpC,IAAA,YAAA,CAAa,gBAAA,CAAiB,kBAAkB,eAAe,CAAA;AAC/D,IAAA,IAAA,CAAK,uBAAA,GAA0B,MAAA,CAAO,WAAA,CAAY,eAAA,EAAiB,EAAE,CAAA;AAAA,EACvE;AAAA,EAEA,aAAa,MAAA,EAA2B;AACtC,IAAA,IAAI,IAAA,CAAK,wBAAwB,MAAA,EAAQ;AACvC,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,mBAAA,GAAsB,MAAA;AAE3B,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA,KAAK,IAAA,CAAK,mBAAA,EAAoB,CAAE,KAAA,CAAM,CAAC,KAAA,KAAU;AAC/C,UAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,QAC9B,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA,CAAK,YAAA,CAAa,uBAAA,CAAwB,MAAM,CAAA;AACnE,MAAA,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,YAAA,CAAa,cAAA,EAAe;AACjD,MAAA,IAAA,CAAK,SAAS,OAAA,GAAU,GAAA;AACxB,MAAA,IAAA,CAAK,SAAS,qBAAA,GAAwB,GAAA;AACtC,MAAA,IAAA,CAAK,WAAA,CAAY,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AAEtC,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,IAC5B,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,UAAA,GAAmB;AACjB,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,KAAA,GAAc;AACZ,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,MAAA,GAAe;AACb,IAAA,IAAI,CAAC,IAAA,CAAK,QAAA,IAAY,IAAA,CAAK,eAAA,EAAiB;AAC1C,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,OAAA,GAAgB;AACd,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAEnB,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,KAAK,IAAA,CAAK,aAAa,KAAA,EAAM;AAC7B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,mBAAA,GAAsB,IAAA;AAC3B,IAAA,IAAA,CAAK,QAAA,GAAW,IAAA;AAAA,EAClB;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,QAAA,CAAS,iBAAA;AACnC,IAAA,MAAM,SAAA,GAAY,IAAI,UAAA,CAAW,YAAY,CAAA;AAE7C,IAAA,IAAA,CAAK,eAAA,GAAkB,MAAA,CAAO,WAAA,CAAY,MAAM;AAC9C,MAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,QAAA,CAAS,qBAAqB,SAAS,CAAA;AAE5C,MAAA,IAAI,GAAA,GAAM,CAAA;AACV,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,YAAA,EAAc,SAAS,CAAA,EAAG;AACpD,QAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAK,CAAA,GAAI,GAAA;AACtC,QAAA,GAAA,IAAO,UAAA,GAAa,UAAA;AAAA,MACtB;AAEA,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,IAAA,CAAK,GAAA,GAAM,YAAY,CAAA;AACxC,MAAA,IAAA,CAAK,QAAQ,KAAA,CAAM,IAAA,CAAK,IAAI,GAAA,GAAM,CAAA,EAAG,CAAC,CAAC,CAAA;AAAA,IACzC,CAAA,EAAG,MAAO,EAAE,CAAA;AAAA,EACd;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,aAAA,CAAc,KAAK,eAAe,CAAA;AAClC,MAAA,IAAA,CAAK,eAAA,GAAkB,IAAA;AAAA,IACzB;AACA,IAAA,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAC,CAAA;AAAA,EACtB;AAAA,EAEQ,uBAAA,GAAgC;AACtC,IAAA,IAAI,KAAK,uBAAA,EAAyB;AAChC,MAAA,aAAA,CAAc,KAAK,uBAAuB,CAAA;AAC1C,MAAA,IAAA,CAAK,uBAAA,GAA0B,IAAA;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,0BAAA,GAAmC;AACzC,IAAA,IAAI,IAAA,CAAK,oBAAA,IAAwB,IAAA,CAAK,4BAAA,EAA8B;AAClE,MAAA,IAAA,CAAK,oBAAA,CAAqB,mBAAA;AAAA,QACxB,gBAAA;AAAA,QACA,IAAA,CAAK;AAAA,OACP;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,IAAA;AAAA,EACtC;AACF;;;ACnKO,IAAM,6BAAA,GAAgC;AACtC,IAAM,6BAAA,GAAgC;AACtC,IAAM,0CAAA,GACX","file":"chunk-3PIMXNQU.mjs","sourcesContent":["export const DEFAULT_MICROPHONE_CONSTRAINTS = {\n echoCancellation: true,\n noiseSuppression: true,\n autoGainControl: true,\n} satisfies MediaTrackConstraints;\n\nexport async function acquireMicrophoneStream(): Promise<MediaStream> {\n return navigator.mediaDevices.getUserMedia({\n audio: DEFAULT_MICROPHONE_CONSTRAINTS,\n });\n}\n","import { subscribeBrowserLifecycle } from \"@charivo/core\";\nimport { acquireMicrophoneStream } from \"./microphone\";\n\nexport const DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1_000;\n\nexport interface TransportLifecycleCallbacks {\n onHidden?(): void;\n onOnline?(): void;\n onPageHide?(): void;\n onPageShow?(event: PageTransitionEvent): void;\n onVisible?(): void;\n onDeviceChange?(): void;\n}\n\nexport function bindTransportLifecycle(\n callbacks: TransportLifecycleCallbacks,\n): () => void {\n const teardownBrowserLifecycle = subscribeBrowserLifecycle({\n onHidden: callbacks.onHidden,\n onOnline: callbacks.onOnline,\n onPageHide: callbacks.onPageHide,\n onPageShow: callbacks.onPageShow,\n onVisible: callbacks.onVisible,\n });\n\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.addEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n\n return () => {\n teardownBrowserLifecycle();\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.removeEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n };\n}\n\nexport interface AudioSenderTrackState {\n audioSender: RTCRtpSender | null;\n mediaStream: MediaStream | null;\n peerConnection: RTCPeerConnection | null;\n}\n\nexport async function replaceMicrophoneTrack(\n state: AudioSenderTrackState,\n): Promise<{ audioSender: RTCRtpSender; mediaStream: MediaStream }> {\n if (!state.peerConnection) {\n throw new Error(\"Realtime transport not active\");\n }\n\n const nextStream = await acquireMicrophoneStream();\n const nextTrack = nextStream.getAudioTracks()[0];\n if (!nextTrack) {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n const sender =\n state.audioSender ??\n (typeof state.peerConnection.getSenders === \"function\"\n ? (state.peerConnection\n .getSenders()\n .find((candidate) => candidate.track?.kind === \"audio\") ?? null)\n : null);\n if (!sender) {\n throw new Error(\"No outbound audio sender available\");\n }\n\n await sender.replaceTrack(nextTrack);\n state.mediaStream?.getTracks().forEach((track) => track.stop());\n\n return {\n audioSender: sender,\n mediaStream: nextStream,\n };\n}\n\nexport function createIceDisconnectDebouncer(\n onDisconnect: () => void,\n delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS,\n): {\n schedule(): void;\n cancel(): void;\n} {\n let pending: ReturnType<typeof setTimeout> | null = null;\n\n return {\n schedule() {\n if (pending) {\n return;\n }\n\n pending = setTimeout(() => {\n pending = null;\n onDisconnect();\n }, delayMs);\n },\n cancel() {\n if (!pending) {\n return;\n }\n\n clearTimeout(pending);\n pending = null;\n },\n };\n}\n","interface LipSyncAnalyzerOptions {\n onRms: (rms: number) => void;\n onError?: (error: unknown) => void;\n}\n\nexport class LipSyncAnalyzer {\n private audioAnalysisStream: MediaStream | null = null;\n private audioContext: AudioContext | null = null;\n private audioSource: MediaStreamAudioSourceNode | null = null;\n private analyser: AnalyserNode | null = null;\n private lipSyncInterval: number | null = null;\n private pendingAudioElementPoll: number | null = null;\n private observedAudioElement: HTMLAudioElement | null = null;\n private observedAudioElementListener: (() => void) | null = null;\n\n constructor(private options: LipSyncAnalyzerOptions) {}\n\n async prepareAudioContext(): Promise<void> {\n if (this.audioContext) {\n return;\n }\n\n const audioContextConstructor =\n window.AudioContext ||\n (window as Window & { webkitAudioContext?: typeof AudioContext })\n .webkitAudioContext;\n if (!audioContextConstructor) {\n throw new Error(\"AudioContext is not supported in this browser\");\n }\n\n this.audioContext = new audioContextConstructor();\n }\n\n observeAudioElement(audioElement: HTMLAudioElement): void {\n this.detachObservedAudioElement();\n\n const tryAttachStream = () => {\n const stream = audioElement.srcObject;\n if (stream instanceof MediaStream) {\n this.attachStream(stream);\n }\n };\n\n this.observedAudioElement = audioElement;\n this.observedAudioElementListener = tryAttachStream;\n audioElement.addEventListener(\"loadedmetadata\", tryAttachStream);\n this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);\n }\n\n attachStream(stream: MediaStream): void {\n if (this.audioAnalysisStream === stream) {\n return;\n }\n\n this.stopAudioElementPolling();\n this.stopLipSyncAnalysis();\n this.audioAnalysisStream = stream;\n\n try {\n if (!this.audioContext) {\n void this.prepareAudioContext().catch((error) => {\n this.options.onError?.(error);\n });\n }\n\n if (!this.audioContext) {\n return;\n }\n\n this.audioSource?.disconnect?.();\n this.audioSource = this.audioContext.createMediaStreamSource(stream);\n this.analyser = this.audioContext.createAnalyser();\n this.analyser.fftSize = 256;\n this.analyser.smoothingTimeConstant = 0.8;\n this.audioSource.connect(this.analyser);\n\n this.startLipSyncAnalysis();\n } catch (error) {\n this.options.onError?.(error);\n }\n }\n\n stopOutput(): void {\n this.stopLipSyncAnalysis();\n }\n\n pause(): void {\n this.stopLipSyncAnalysis();\n }\n\n resume(): void {\n if (!this.analyser || this.lipSyncInterval) {\n return;\n }\n\n this.startLipSyncAnalysis();\n }\n\n cleanup(): void {\n this.stopLipSyncAnalysis();\n this.stopAudioElementPolling();\n this.detachObservedAudioElement();\n\n this.audioSource?.disconnect?.();\n this.audioSource = null;\n\n if (this.audioContext) {\n void this.audioContext.close();\n this.audioContext = null;\n }\n\n this.audioAnalysisStream = null;\n this.analyser = null;\n }\n\n private startLipSyncAnalysis(): void {\n if (!this.analyser) {\n return;\n }\n\n const bufferLength = this.analyser.frequencyBinCount;\n const dataArray = new Uint8Array(bufferLength);\n\n this.lipSyncInterval = window.setInterval(() => {\n if (!this.analyser) {\n return;\n }\n\n this.analyser.getByteFrequencyData(dataArray);\n\n let sum = 0;\n for (let index = 0; index < bufferLength; index += 1) {\n const normalized = dataArray[index] / 255;\n sum += normalized * normalized;\n }\n\n const rms = Math.sqrt(sum / bufferLength);\n this.options.onRms(Math.min(rms * 3, 1));\n }, 1000 / 60);\n }\n\n private stopLipSyncAnalysis(): void {\n if (this.lipSyncInterval) {\n clearInterval(this.lipSyncInterval);\n this.lipSyncInterval = null;\n }\n this.options.onRms(0);\n }\n\n private stopAudioElementPolling(): void {\n if (this.pendingAudioElementPoll) {\n clearInterval(this.pendingAudioElementPoll);\n this.pendingAudioElementPoll = null;\n }\n }\n\n private detachObservedAudioElement(): void {\n if (this.observedAudioElement && this.observedAudioElementListener) {\n this.observedAudioElement.removeEventListener(\n \"loadedmetadata\",\n this.observedAudioElementListener,\n );\n }\n\n this.observedAudioElement = null;\n this.observedAudioElementListener = null;\n }\n}\n","import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS as DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS } from \"../instructions\";\n\n// Mirrors packages/server/src/openai/realtime/index.ts until the server/browser\n// OpenAI defaults consolidate under ROADMAP P0-ARCH-2.\nexport const DEFAULT_OPENAI_REALTIME_MODEL = \"gpt-realtime-mini\";\nexport const DEFAULT_OPENAI_REALTIME_VOICE = \"marin\";\nexport const DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS =\n DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS;\n"]}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { delay } from './chunk-XPKFJWHL.mjs';
|
|
2
|
+
import { LipSyncAnalyzer, createIceDisconnectDebouncer, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, acquireMicrophoneStream, bindTransportLifecycle, replaceMicrophoneTrack, DEFAULT_OPENAI_REALTIME_VOICE } from './chunk-3PIMXNQU.mjs';
|
|
2
3
|
import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-PNBYFAER.mjs';
|
|
3
4
|
import { OPENAI_REALTIME_ADAPTER } from '@charivo/core';
|
|
4
5
|
|
|
@@ -21,10 +22,10 @@ var OpenAIRealtimeClient = class {
|
|
|
21
22
|
audioElement = null;
|
|
22
23
|
mediaStream = null;
|
|
23
24
|
audioSender = null;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
26
|
+
onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
|
|
27
|
+
onError: (error) => console.error("Failed to setup audio analysis:", error)
|
|
28
|
+
});
|
|
28
29
|
isResponseInProgress = false;
|
|
29
30
|
cancelInFlight = false;
|
|
30
31
|
hasStartedAssistantResponse = false;
|
|
@@ -56,14 +57,14 @@ var OpenAIRealtimeClient = class {
|
|
|
56
57
|
this.audioElement = document.createElement("audio");
|
|
57
58
|
this.audioElement.autoplay = true;
|
|
58
59
|
this.audioElement.setAttribute("playsinline", "true");
|
|
59
|
-
this.
|
|
60
|
+
void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
|
|
60
61
|
this.bindConnectionEvents();
|
|
61
62
|
this.bindTransportLifecycleEvents();
|
|
62
63
|
this.pc.ontrack = (event) => {
|
|
63
64
|
if (this.audioElement) {
|
|
64
65
|
this.audioElement.srcObject = event.streams[0];
|
|
65
66
|
}
|
|
66
|
-
this.
|
|
67
|
+
this.lipSyncAnalyzer.attachStream(event.streams[0]);
|
|
67
68
|
};
|
|
68
69
|
try {
|
|
69
70
|
this.mediaStream = await acquireMicrophoneStream();
|
|
@@ -90,14 +91,10 @@ var OpenAIRealtimeClient = class {
|
|
|
90
91
|
session: config ?? {},
|
|
91
92
|
sdpOffer: offer.sdp
|
|
92
93
|
});
|
|
93
|
-
|
|
94
|
-
throw new Error(
|
|
95
|
-
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
96
|
-
);
|
|
97
|
-
}
|
|
94
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
98
95
|
await this.pc.setRemoteDescription({
|
|
99
96
|
type: "answer",
|
|
100
|
-
sdp:
|
|
97
|
+
sdp: answerSdp
|
|
101
98
|
});
|
|
102
99
|
this.emitEvent({ type: "session.started" });
|
|
103
100
|
} catch (error) {
|
|
@@ -286,10 +283,7 @@ var OpenAIRealtimeClient = class {
|
|
|
286
283
|
if (this.cancelInFlight || !event.delta) {
|
|
287
284
|
return;
|
|
288
285
|
}
|
|
289
|
-
|
|
290
|
-
this.hasStartedAssistantResponse = true;
|
|
291
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
292
|
-
}
|
|
286
|
+
this.ensureAssistantResponseStarted();
|
|
293
287
|
this.assistantText += event.delta;
|
|
294
288
|
this.emitEvent({
|
|
295
289
|
type: "assistant.text.delta",
|
|
@@ -300,39 +294,13 @@ var OpenAIRealtimeClient = class {
|
|
|
300
294
|
if (this.cancelInFlight || !event.text) {
|
|
301
295
|
return;
|
|
302
296
|
}
|
|
303
|
-
|
|
304
|
-
this.hasStartedAssistantResponse = true;
|
|
305
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
306
|
-
}
|
|
307
|
-
if (event.text !== this.assistantText) {
|
|
308
|
-
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
309
|
-
this.assistantText = event.text;
|
|
310
|
-
if (delta) {
|
|
311
|
-
this.emitEvent({
|
|
312
|
-
type: "assistant.text.delta",
|
|
313
|
-
text: delta
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
}
|
|
297
|
+
this.emitFinalAssistantText(event.text);
|
|
317
298
|
return;
|
|
318
299
|
case "response.output_audio_transcript.done":
|
|
319
300
|
if (this.cancelInFlight || !event.transcript) {
|
|
320
301
|
return;
|
|
321
302
|
}
|
|
322
|
-
|
|
323
|
-
this.hasStartedAssistantResponse = true;
|
|
324
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
325
|
-
}
|
|
326
|
-
if (event.transcript !== this.assistantText) {
|
|
327
|
-
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
328
|
-
this.assistantText = event.transcript;
|
|
329
|
-
if (delta) {
|
|
330
|
-
this.emitEvent({
|
|
331
|
-
type: "assistant.text.delta",
|
|
332
|
-
text: delta
|
|
333
|
-
});
|
|
334
|
-
}
|
|
335
|
-
}
|
|
303
|
+
this.emitFinalAssistantText(event.transcript);
|
|
336
304
|
return;
|
|
337
305
|
case "conversation.item.input_audio_transcription.completed":
|
|
338
306
|
if (event.transcript) {
|
|
@@ -463,17 +431,17 @@ var OpenAIRealtimeClient = class {
|
|
|
463
431
|
this.emitConnectionLost("online");
|
|
464
432
|
};
|
|
465
433
|
handleHidden = () => {
|
|
466
|
-
this.
|
|
434
|
+
this.lipSyncAnalyzer.pause();
|
|
467
435
|
};
|
|
468
436
|
handleVisible = () => {
|
|
469
|
-
this.
|
|
437
|
+
this.lipSyncAnalyzer.resume();
|
|
470
438
|
this.emitConnectionLost("visibility");
|
|
471
439
|
};
|
|
472
440
|
handlePageHide = () => {
|
|
473
|
-
this.
|
|
441
|
+
this.lipSyncAnalyzer.pause();
|
|
474
442
|
};
|
|
475
443
|
handlePageShow = (event) => {
|
|
476
|
-
this.
|
|
444
|
+
this.lipSyncAnalyzer.resume();
|
|
477
445
|
if (event.persisted) {
|
|
478
446
|
this.emitConnectionLost("pageshow");
|
|
479
447
|
}
|
|
@@ -508,7 +476,7 @@ var OpenAIRealtimeClient = class {
|
|
|
508
476
|
this.isResponseInProgress = false;
|
|
509
477
|
this.cancelInFlight = false;
|
|
510
478
|
this.resetResponseTracking();
|
|
511
|
-
this.
|
|
479
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
512
480
|
this.emitEvent({
|
|
513
481
|
type: "connection.lost",
|
|
514
482
|
cause,
|
|
@@ -524,77 +492,11 @@ var OpenAIRealtimeClient = class {
|
|
|
524
492
|
}
|
|
525
493
|
return true;
|
|
526
494
|
}
|
|
527
|
-
prepareAudioAnalysis() {
|
|
528
|
-
if (this.audioContext) {
|
|
529
|
-
return;
|
|
530
|
-
}
|
|
531
|
-
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
532
|
-
if (!audioContextConstructor) {
|
|
533
|
-
return;
|
|
534
|
-
}
|
|
535
|
-
this.audioContext = new audioContextConstructor();
|
|
536
|
-
}
|
|
537
|
-
setupAudioAnalysis(stream) {
|
|
538
|
-
try {
|
|
539
|
-
this.prepareAudioAnalysis();
|
|
540
|
-
if (!this.audioContext) {
|
|
541
|
-
return;
|
|
542
|
-
}
|
|
543
|
-
this.audioSource?.disconnect();
|
|
544
|
-
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
545
|
-
this.analyser = this.audioContext.createAnalyser();
|
|
546
|
-
this.analyser.fftSize = 256;
|
|
547
|
-
this.analyser.smoothingTimeConstant = 0.8;
|
|
548
|
-
this.audioSource.connect(this.analyser);
|
|
549
|
-
this.startLipSyncAnalysis();
|
|
550
|
-
} catch (error) {
|
|
551
|
-
console.error("Failed to setup audio analysis:", error);
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
startLipSyncAnalysis() {
|
|
555
|
-
if (!this.analyser) {
|
|
556
|
-
return;
|
|
557
|
-
}
|
|
558
|
-
const bufferLength = this.analyser.frequencyBinCount;
|
|
559
|
-
const dataArray = new Uint8Array(bufferLength);
|
|
560
|
-
this.lipSyncInterval = window.setInterval(() => {
|
|
561
|
-
if (!this.analyser) {
|
|
562
|
-
return;
|
|
563
|
-
}
|
|
564
|
-
this.analyser.getByteFrequencyData(dataArray);
|
|
565
|
-
let sum = 0;
|
|
566
|
-
for (let index = 0; index < bufferLength; index += 1) {
|
|
567
|
-
const normalized = dataArray[index] / 255;
|
|
568
|
-
sum += normalized * normalized;
|
|
569
|
-
}
|
|
570
|
-
const rms = Math.sqrt(sum / bufferLength);
|
|
571
|
-
this.emitEvent({
|
|
572
|
-
type: "audio.lipsync",
|
|
573
|
-
rms: Math.min(rms * 3, 1)
|
|
574
|
-
});
|
|
575
|
-
}, 1e3 / 60);
|
|
576
|
-
}
|
|
577
|
-
stopLipSyncAnalysis() {
|
|
578
|
-
if (this.lipSyncInterval) {
|
|
579
|
-
clearInterval(this.lipSyncInterval);
|
|
580
|
-
this.lipSyncInterval = null;
|
|
581
|
-
}
|
|
582
|
-
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
583
|
-
}
|
|
584
|
-
pauseAudioAnalysis() {
|
|
585
|
-
this.stopLipSyncAnalysis();
|
|
586
|
-
}
|
|
587
|
-
resumeAudioAnalysis() {
|
|
588
|
-
if (!this.analyser || this.lipSyncInterval) {
|
|
589
|
-
return;
|
|
590
|
-
}
|
|
591
|
-
this.startLipSyncAnalysis();
|
|
592
|
-
}
|
|
593
495
|
cleanup() {
|
|
594
496
|
this.isCleaningUp = true;
|
|
595
497
|
this.unbindTransportLifecycleEvents();
|
|
596
498
|
this.iceDisconnectDebouncer.cancel();
|
|
597
|
-
this.
|
|
499
|
+
this.lipSyncAnalyzer.cleanup();
|
|
598
500
|
this.rejectPendingSessionUpdate(
|
|
599
501
|
new Error("Realtime session ended before session update completed")
|
|
600
502
|
);
|
|
@@ -614,13 +516,7 @@ var OpenAIRealtimeClient = class {
|
|
|
614
516
|
this.audioElement.srcObject = null;
|
|
615
517
|
this.audioElement = null;
|
|
616
518
|
}
|
|
617
|
-
if (this.audioContext) {
|
|
618
|
-
this.audioContext.close();
|
|
619
|
-
this.audioContext = null;
|
|
620
|
-
}
|
|
621
|
-
this.audioSource = null;
|
|
622
519
|
this.audioSender = null;
|
|
623
|
-
this.analyser = null;
|
|
624
520
|
this.isResponseInProgress = false;
|
|
625
521
|
this.cancelInFlight = false;
|
|
626
522
|
this.connectionLossNotified = false;
|
|
@@ -668,14 +564,10 @@ var OpenAIRealtimeClient = class {
|
|
|
668
564
|
session: config ?? this.currentSessionConfig ?? {},
|
|
669
565
|
sdpOffer: offer.sdp
|
|
670
566
|
});
|
|
671
|
-
|
|
672
|
-
throw new Error(
|
|
673
|
-
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
674
|
-
);
|
|
675
|
-
}
|
|
567
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
676
568
|
await pc.setRemoteDescription({
|
|
677
569
|
type: "answer",
|
|
678
|
-
sdp:
|
|
570
|
+
sdp: answerSdp
|
|
679
571
|
});
|
|
680
572
|
return true;
|
|
681
573
|
}
|
|
@@ -713,11 +605,38 @@ var OpenAIRealtimeClient = class {
|
|
|
713
605
|
getSessionUpdateTimeoutMs() {
|
|
714
606
|
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
715
607
|
}
|
|
608
|
+
resolveWebRTCAnswerSdp(bootstrap) {
|
|
609
|
+
if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
610
|
+
throw new Error(
|
|
611
|
+
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
return bootstrap.answerSdp;
|
|
615
|
+
}
|
|
716
616
|
resetResponseTracking() {
|
|
717
617
|
this.assistantText = "";
|
|
718
618
|
this.hasStartedAssistantResponse = false;
|
|
719
619
|
this.hasStartedAudioOutput = false;
|
|
720
620
|
}
|
|
621
|
+
ensureAssistantResponseStarted() {
|
|
622
|
+
if (!this.hasStartedAssistantResponse) {
|
|
623
|
+
this.hasStartedAssistantResponse = true;
|
|
624
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
emitFinalAssistantText(full) {
|
|
628
|
+
this.ensureAssistantResponseStarted();
|
|
629
|
+
if (full !== this.assistantText) {
|
|
630
|
+
const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
|
|
631
|
+
this.assistantText = full;
|
|
632
|
+
if (delta) {
|
|
633
|
+
this.emitEvent({
|
|
634
|
+
type: "assistant.text.delta",
|
|
635
|
+
text: delta
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
721
640
|
emitEvent(event) {
|
|
722
641
|
for (const callback of this.eventCallbacks) {
|
|
723
642
|
callback(event);
|
|
@@ -770,12 +689,7 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
770
689
|
}
|
|
771
690
|
return session;
|
|
772
691
|
}
|
|
773
|
-
function delay(ms) {
|
|
774
|
-
return new Promise((resolve) => {
|
|
775
|
-
setTimeout(resolve, ms);
|
|
776
|
-
});
|
|
777
|
-
}
|
|
778
692
|
|
|
779
693
|
export { OpenAIRealtimeClient, createOpenAIRealtimeClient };
|
|
780
|
-
//# sourceMappingURL=chunk-
|
|
781
|
-
//# sourceMappingURL=chunk-
|
|
694
|
+
//# sourceMappingURL=chunk-MI4QTRWT.mjs.map
|
|
695
|
+
//# sourceMappingURL=chunk-MI4QTRWT.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/openai/client.ts"],"names":[],"mappings":";;;;;AAwEA,IAAM,qBAAA,uBAA4B,GAAA,CAAY;AAAA,EAC5C,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,kBAAA;AAAA,EACA,eAAA;AAAA,EACA,uCAAA;AAAA,EACA,uDAAA;AAAA,EACA;AACF,CAAC,CAAA;AACD,IAAM,iCAAA,GAAoC,GAAA;AAQnC,IAAM,uBAAN,MAA8D;AAAA,EA4BnE,WAAA,CAAoB,OAAA,GAAuC,EAAC,EAAG;AAA3C,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAA4C;AAAA,EA3BxD,EAAA,GAA+B,IAAA;AAAA,EAC/B,EAAA,GAA4B,IAAA;AAAA,EAC5B,YAAA,GAAwC,IAAA;AAAA,EACxC,WAAA,GAAkC,IAAA;AAAA,EAClC,WAAA,GAAmC,IAAA;AAAA,EAC1B,eAAA,GAAkB,IAAI,eAAA,CAAgB;AAAA,IACrD,KAAA,EAAO,CAAC,GAAA,KAAQ,IAAA,CAAK,UAAU,EAAE,IAAA,EAAM,eAAA,EAAiB,GAAA,EAAK,CAAA;AAAA,IAC7D,SAAS,CAAC,KAAA,KAAU,OAAA,CAAQ,KAAA,CAAM,mCAAmC,KAAK;AAAA,GAC3E,CAAA;AAAA,EACO,oBAAA,GAAuB,KAAA;AAAA,EACvB,cAAA,GAAiB,KAAA;AAAA,EACjB,2BAAA,GAA8B,KAAA;AAAA,EAC9B,qBAAA,GAAwB,KAAA;AAAA,EACxB,aAAA,GAAgB,EAAA;AAAA,EAChB,mBAAA,GAAsB,CAAA;AAAA,EACtB,oBAAA,GAAoD,IAAA;AAAA,EACpD,oBAAA;AAAA,EACA,sBAAA,GAAyB,KAAA;AAAA,EACzB,oBAAA,GAAuB,KAAA;AAAA,EACvB,YAAA,GAAe,KAAA;AAAA,EACf,YAAA,GAAe,KAAA;AAAA,EACf,0BAAA;AAAA,EACS,sBAAA,GAAyB,6BAA6B,MAAM;AAC3E,IAAA,IAAA,CAAK,mBAAmB,kBAAkB,CAAA;AAAA,EAC5C,GAAG,oCAAoC,CAAA;AAAA,EAC/B,cAAA,uBAAqB,GAAA,EAA6C;AAAA,EAI1E,MAAM,QAAQ,MAAA,EAA+C;AAC3D,IAAA,IAAI;AACF,MAAA,IAAI,IAAA,CAAK,oBAAA,IAAwB,IAAA,CAAK,YAAA,EAAc;AAClD,QAAA,MAAM,IAAI,MAAM,uCAAuC,CAAA;AAAA,MACzD;AAEA,MAAA,IAAA,CAAK,IAAI,4CAA4C,CAAA;AAErD,MAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,MAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,MAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,MAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAC5B,MAAA,IAAA,CAAK,EAAA,GAAK,IAAI,iBAAA,EAAkB;AAChC,MAAA,IAAA,CAAK,YAAA,GAAe,QAAA,CAAS,aAAA,CAAc,OAAO,CAAA;AAClD,MAAA,IAAA,CAAK,aAAa,QAAA,GAAW,IAAA;AAC7B,MAAA,IAAA,CAAK,YAAA,CAAa,YAAA,CAAa,aAAA,EAAe,MAAM,CAAA;AACpD,MAAA,KAAK,KAAK,eAAA,CAAgB,mBAAA,EAAoB,CAAE,KAAA,CAAM,MAAM,KAAA,CAAS,CAAA;AACrE,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAC1B,MAAA,IAAA,CAAK,4BAAA,EAA6B;AAElC,MAAA,IAAA,CAAK,EAAA,CAAG,OAAA,GAAU,CAAC,KAAA,KAAU;AAC3B,QAAA,IAAI,KAAK,YAAA,EAAc;AACrB,UAAA,IAAA,CAAK,YAAA,CAAa,SAAA,GAAY,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAA;AAAA,QAC/C;AACA,QAAA,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA;AAAA,MACpD,CAAA;AAEA,MAAA,IAAI;AACF,QAAA,IAAA,CAAK,WAAA,GAAc,MAAM,uBAAA,EAAwB;AACjD,QAAA,MAAM,UAAA,GAAa,IAAA,CAAK,WAAA,CAAY,SAAA,GAAY,CAAC,CAAA;AACjD,QAAA,IAAA,CAAK,WAAA,GACH,KAAK,EAAA,CAAG,QAAA,CAAS,YAAY,IAAA,CAAK,WAAW,KAAK,IAAA,CAAK,WAAA;AAAA,MAC3D,CAAA,CAAA,MAAQ;AACN,QAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,MAC/D;AAEA,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA,CAAK,EAAA,CAAG,iBAAA,CAAkB,YAAY,CAAA;AAChD,MAAA,IAAA,CAAK,EAAA,CAAG,SAAA,GAAY,CAAC,KAAA,KAAU;AAC7B,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,KAAA,CAAM,KAAA,CAAM,IAAI,CAAA;AACrC,QAAA,IAAA,CAAK,kBAAkB,OAAO,CAAA;AAAA,MAChC,CAAA;AACA,MAAA,IAAA,CAAK,EAAA,CAAG,UAAU,MAAM;AACtB,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,OAAA;AAAA,UACN,KAAA,EAAO,IAAI,KAAA,CAAM,mBAAmB;AAAA,SACrC,CAAA;AAAA,MACH,CAAA;AAEA,MAAA,MAAM,KAAA,GAAQ,MAAM,IAAA,CAAK,EAAA,CAAG,WAAA,EAAY;AACxC,MAAA,MAAM,IAAA,CAAK,EAAA,CAAG,mBAAA,CAAoB,KAAK,CAAA;AAEvC,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,mBAAA,CAAoB;AAAA,QAC/C,SAAA,EAAW,QAAA;AAAA,QACX,OAAA,EAAS,UAAU,EAAC;AAAA,QACpB,UAAU,KAAA,CAAM;AAAA,OACjB,CAAA;AAED,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,sBAAA,CAAuB,SAAS,CAAA;AAEvD,MAAA,MAAM,IAAA,CAAK,GAAG,oBAAA,CAAqB;AAAA,QACjC,IAAA,EAAM,QAAA;AAAA,QACN,GAAA,EAAK;AAAA,OACN,CAAA;AAED,MAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAA;AAAA,IAC5C,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,EAAQ;AACb,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,IAAI,sCAAsC,CAAA;AAC/C,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,IAAA,IAAA,CAAK,OAAA,EAAQ;AAAA,EACf;AAAA,EAEA,MAAM,cAAc,MAAA,EAA+C;AACjE,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AACvC,IAAA,MAAM,OAAA,GAAU,CAAA,uBAAA,EAA0B,IAAA,CAAK,mBAAA,EAAqB,CAAA,CAAA;AAEpE,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,IAC/D;AAEA,IAAA,EAAA,CAAG,IAAA;AAAA,MACD,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,gBAAA;AAAA,QACN,QAAA,EAAU,OAAA;AAAA,QACV,OAAA,EAAS,8BAA8B,MAAM;AAAA,OAC9C;AAAA,KACH;AAEA,IAAA,MAAM,IAAI,OAAA,CAAc,CAAC,OAAA,EAAS,MAAA,KAAW;AAC3C,MAAA,MAAM,SAAA,GAAY,WAAW,MAAM;AACjC,QAAA,IAAA,CAAK,0BAAA;AAAA,UACH,IAAI,KAAA;AAAA,YACF,CAAA,4CAAA,EAA+C,IAAA,CAAK,yBAAA,EAA2B,CAAA,EAAA;AAAA;AACjF,SACF;AAAA,MACF,CAAA,EAAG,IAAA,CAAK,yBAAA,EAA2B,CAAA;AAEnC,MAAA,IAAA,CAAK,oBAAA,GAAuB;AAAA,QAC1B,OAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAAA,EAC9B;AAAA,EAEA,MAAM,QAAQ,MAAA,EAA+C;AAC3D,IAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AACpB,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,MAAA;AAE5B,IAAA,IAAI;AACF,MAAA,IAAI,MAAM,IAAA,CAAK,qBAAA,CAAsB,MAAM,CAAA,EAAG;AAC5C,QAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,IAAA,CAAK,kBAAkB,MAAM,CAAA;AACnC,MAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAAA,IAChC,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,IAAA,EAA6B;AAC1C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,IAAI,KAAK,oBAAA,EAAsB;AAC7B,MAAA,OAAA,CAAQ,KAAK,6DAAmD,CAAA;AAChE,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,EAAA,CAAG,IAAA;AAAA,MACN,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,SAAA;AAAA,UACN,IAAA,EAAM,MAAA;AAAA,UACN,OAAA,EAAS;AAAA,YACP;AAAA,cACE,IAAA,EAAM,YAAA;AAAA,cACN;AAAA;AACF;AACF;AACF,OACD;AAAA,KACH;AACA,IAAA,IAAA,CAAK,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAExD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,UAAU,MAAA,EAAoC;AAClD,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN;AAAA,KACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAA,CACJ,MAAA,EACA,MAAA,EACe;AACf,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,EAAA,CAAG,IAAA;AAAA,MACD,KAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,0BAAA;AAAA,QACN,IAAA,EAAM;AAAA,UACJ,IAAA,EAAM,sBAAA;AAAA,UACN,OAAA,EAAS,MAAA;AAAA,UACT,MAAA,EAAQ,IAAA,CAAK,SAAA,CAAU,MAAM;AAAA;AAC/B,OACD;AAAA,KACH;AACA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AAEnD,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,SAAA,GAA2B;AAC/B,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,EAAA,GAAK,KAAK,sBAAA,EAAuB;AAEvC,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,EAAA,CAAG,KAAK,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,iBAAA,EAAmB,CAAC,CAAA;AACnD,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,IAAA;AAAA,EACxB;AAAA,EAEA,QAAQ,QAAA,EAAyD;AAC/D,IAAA,IAAA,CAAK,cAAA,CAAe,IAAI,QAAQ,CAAA;AAAA,EAClC;AAAA,EAEQ,kBAAkB,KAAA,EAA0B;AAClD,IAAA,IAAI,KAAK,OAAA,CAAQ,KAAA,IAAS,KAAK,mBAAA,CAAoB,KAAA,CAAM,IAAI,CAAA,EAAG;AAC9D,MAAA,IAAA,CAAK,GAAA,CAAI,mCAAA,EAA8B,KAAA,CAAM,IAAA,EAAM,KAAK,CAAA;AAAA,IAC1D;AAEA,IAAA,QAAQ,MAAM,IAAA;AAAM,MAClB,KAAK,iBAAA;AACH,QAAA;AAAA,MAEF,KAAK,iBAAA;AACH,QAAA,IAAA,CAAK,2BAAA,EAA4B;AACjC,QAAA;AAAA,MAEF,KAAK,sBAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA;AAAA,QACF;AAEA,QAAA,IAAI,CAAC,KAAK,qBAAA,EAAuB;AAC/B,UAAA,IAAA,CAAK,qBAAA,GAAwB,IAAA;AAC7B,UAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,sBAAA,EAAwB,CAAA;AAAA,QACjD;AACA,QAAA;AAAA,MAEF,KAAK,qBAAA;AAAA,MACL,KAAK,4BAAA;AAAA,MACL,KAAK,6BAAA;AACH,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,oBAAA,EAAsB,CAAA;AAC7C,QAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,eAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAI,KAAK,cAAA,EAAgB;AACvB,UAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,UAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,UAAA;AAAA,QACF;AAOA,QAAA,IAAI,CAAC,KAAK,aAAA,EAAe;AACvB,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,8BAAA;AAAA,UACN,MAAM,IAAA,CAAK,aAAA;AAAA,UACX,KAAA,EAAO,MAAM,QAAA,EAAU,KAAA;AAAA,UACvB,KAAA,EAAO,MAAM,QAAA,EAAU,KAAA;AAAA,UACvB,UAAA,EAAY,MAAM,QAAA,EAAU;AAAA,SAC7B,CAAA;AACD,QAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,QAAA;AAAA,MAEF,KAAK,iCAAA;AAAA,MACL,KAAK,wCAAA;AAAA,MACL,KAAK,4BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,KAAA,EAAO;AACvC,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,8BAAA,EAA+B;AAEpC,QAAA,IAAA,CAAK,iBAAiB,KAAA,CAAM,KAAA;AAC5B,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,sBAAA;AAAA,UACN,MAAM,KAAA,CAAM;AAAA,SACb,CAAA;AACD,QAAA;AAAA,MAEF,KAAK,2BAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,IAAA,EAAM;AACtC,UAAA;AAAA,QACF;AACA,QAAA,IAAA,CAAK,sBAAA,CAAuB,MAAM,IAAI,CAAA;AACtC,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAI,IAAA,CAAK,cAAA,IAAkB,CAAC,KAAA,CAAM,UAAA,EAAY;AAC5C,UAAA;AAAA,QACF;AACA,QAAA,IAAA,CAAK,sBAAA,CAAuB,MAAM,UAAU,CAAA;AAC5C,QAAA;AAAA,MAEF,KAAK,uDAAA;AACH,QAAA,IAAI,MAAM,UAAA,EAAY;AACpB,UAAA,IAAA,CAAK,SAAA,CAAU;AAAA,YACb,IAAA,EAAM,iBAAA;AAAA,YACN,MAAM,KAAA,CAAM;AAAA,WACb,CAAA;AAAA,QACH;AACA,QAAA;AAAA,MAEF,KAAK,uCAAA;AACH,QAAA,IAAA,CAAK,mBAAmB,KAAK,CAAA;AAC7B,QAAA;AAAA,MAEF,KAAK,OAAA;AACH,QAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,QAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,QAAA;AACE,UAAA,MAAM,QAAQ,IAAI,KAAA,CAAM,KAAA,CAAM,KAAA,EAAO,WAAW,eAAe,CAAA;AAC/D,UAAA,IAAI,IAAA,CAAK,2BAAA,CAA4B,KAAK,CAAA,EAAG;AAC3C,YAAA,IAAA,CAAK,2BAA2B,KAAK,CAAA;AAAA,UACvC;AAEA,UAAA,IAAA,CAAK,SAAA,CAAU;AAAA,YACb,IAAA,EAAM,OAAA;AAAA,YACN;AAAA,WACD,CAAA;AAAA,QACH;AACA,QAAA;AAAA,MAEF;AACE,QAAA;AAAA;AACJ,EACF;AAAA,EAEQ,mBAAmB,KAAA,EAA0B;AACnD,IAAA,MAAM,SAAS,KAAA,CAAM,OAAA,IAAW,KAAA,CAAM,IAAA,EAAM,WAAW,KAAA,CAAM,OAAA;AAC7D,IAAA,MAAM,IAAA,GAAO,KAAA,CAAM,IAAA,IAAQ,KAAA,CAAM,IAAA,EAAM,IAAA;AACvC,IAAA,MAAM,QAAA,GAAW,KAAA,CAAM,SAAA,IAAa,KAAA,CAAM,IAAA,EAAM,SAAA;AAEhD,IAAA,IAAI,CAAC,IAAA,IAAQ,CAAC,QAAA,EAAU;AACtB,MAAA,OAAA,CAAQ,KAAK,2DAAiD,CAAA;AAC9D,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,OAAgC,EAAC;AACrC,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,QAAQ,CAAA;AAClC,MAAA,IAAI,QAAA,CAAS,MAAM,CAAA,EAAG;AACpB,QAAA,IAAA,GAAO,MAAA;AAAA,MACT;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,gCAAA,EAAkC,KAAA,EAAO,QAAQ,CAAA;AAC/D,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,SAAA,CAAU;AAAA,MACb,IAAA,EAAM,WAAA;AAAA,MACN,IAAA;AAAA,MACA,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,oBACZ,OAAA,EACmC;AACnC,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,OAAO,IAAA,CAAK,OAAA,CAAQ,gBAAA,CAAiB,OAAO,CAAA;AAAA,IAC9C;AAEA,IAAA,MAAM,WAAA,GAAc,KAAK,OAAA,CAAQ,WAAA;AACjC,IAAA,IAAI,CAAC,WAAA,EAAa;AAChB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,MAAM,WAAW,MAAM,gBAAA;AAAA,MACrB,WAAA;AAAA,MACA;AAAA,QACE,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB;AAAA,SAClB;AAAA,QACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,OAAO;AAAA,OAC9B;AAAA,MACA,4CAA4C,0BAA0B,CAAA,EAAA;AAAA,KACxE;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mCAAA,EAAsC,SAAS,CAAA,CAAE,CAAA;AAAA,IACnE;AAEA,IAAA,MAAM,SAAA,GAAa,MAAM,QAAA,CAAS,IAAA,EAAK;AACvC,IAAA,IAAI,CAAC,0BAAA,CAA2B,SAAS,CAAA,EAAG;AAC1C,MAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,IAC/D;AAEA,IAAA,OAAO,SAAA;AAAA,EACT;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,EAAA,EAAI;AACZ,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,EAAA,CAAG,gBAAA,CAAiB,0BAAA,EAA4B,MAAM;AACzD,MAAA,MAAM,QAAA,GAAW,KAAK,EAAA,EAAI,kBAAA;AAC1B,MAAA,IAAI,aAAa,QAAA,EAAU;AACzB,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,QAAA,IAAA,CAAK,mBAAmB,YAAY,CAAA;AAAA,MACtC,CAAA,MAAA,IAAW,aAAa,cAAA,EAAgB;AACtC,QAAA,IAAA,CAAK,uBAAuB,QAAA,EAAS;AAAA,MACvC,CAAA,MAAO;AACL,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AAAA,MACrC;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,EAAA,CAAG,gBAAA,CAAiB,uBAAA,EAAyB,MAAM;AACtD,MAAA,MAAM,eAAA,GAAkB,KAAK,EAAA,EAAI,eAAA;AACjC,MAAA,IAAI,eAAA,KAAoB,QAAA,IAAY,eAAA,KAAoB,QAAA,EAAU;AAChE,QAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,QAAA,IAAA,CAAK,mBAAmB,mBAAmB,CAAA;AAAA,MAC7C;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA,EAEQ,4BAAA,GAAqC;AAC3C,IAAA,IAAI,KAAK,0BAAA,EAA4B;AACnC,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,6BAA6B,sBAAA,CAAuB;AAAA,MACvD,UAAU,IAAA,CAAK,YAAA;AAAA,MACf,UAAU,IAAA,CAAK,YAAA;AAAA,MACf,YAAY,IAAA,CAAK,cAAA;AAAA,MACjB,YAAY,IAAA,CAAK,cAAA;AAAA,MACjB,WAAW,IAAA,CAAK,aAAA;AAAA,MAChB,gBAAgB,IAAA,CAAK;AAAA,KACtB,CAAA;AAAA,EACH;AAAA,EAEQ,8BAAA,GAAuC;AAC7C,IAAA,IAAA,CAAK,0BAAA,IAA6B;AAClC,IAAA,IAAA,CAAK,0BAAA,GAA6B,MAAA;AAAA,EACpC;AAAA,EAEiB,eAAe,MAAY;AAC1C,IAAA,IAAA,CAAK,mBAAmB,QAAQ,CAAA;AAAA,EAClC,CAAA;AAAA,EAEiB,eAAe,MAAY;AAC1C,IAAA,IAAA,CAAK,gBAAgB,KAAA,EAAM;AAAA,EAC7B,CAAA;AAAA,EAEiB,gBAAgB,MAAY;AAC3C,IAAA,IAAA,CAAK,gBAAgB,MAAA,EAAO;AAC5B,IAAA,IAAA,CAAK,mBAAmB,YAAY,CAAA;AAAA,EACtC,CAAA;AAAA,EAEiB,iBAAiB,MAAY;AAC5C,IAAA,IAAA,CAAK,gBAAgB,KAAA,EAAM;AAAA,EAC7B,CAAA;AAAA,EAEiB,cAAA,GAAiB,CAAC,KAAA,KAAqC;AACtE,IAAA,IAAA,CAAK,gBAAgB,MAAA,EAAO;AAC5B,IAAA,IAAI,MAAM,SAAA,EAAW;AACnB,MAAA,IAAA,CAAK,mBAAmB,UAAU,CAAA;AAAA,IACpC;AAAA,EACF,CAAA;AAAA,EAEiB,qBAAqB,MAAY;AAChD,IAAA,KAAK,KAAK,sBAAA,EAAuB;AAAA,EACnC,CAAA;AAAA,EAEA,MAAc,sBAAA,GAAwC;AACpD,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,oBAAA,EAAsB;AACzC,MAAA;AAAA,IACF;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,EAAE,WAAA,EAAa,WAAA,EAAY,GAAI,MAAM,sBAAA,CAAuB;AAAA,QAChE,aAAa,IAAA,CAAK,WAAA;AAAA,QAClB,aAAa,IAAA,CAAK,WAAA;AAAA,QAClB,gBAAgB,IAAA,CAAK;AAAA,OACtB,CAAA;AACD,MAAA,IAAA,CAAK,WAAA,GAAc,WAAA;AACnB,MAAA,IAAA,CAAK,WAAA,GAAc,WAAA;AAAA,IACrB,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,SAAA,CAAU;AAAA,QACb,IAAA,EAAM,OAAA;AAAA,QACN,KAAA,EAAO,iBAAiB,KAAA,GAAQ,KAAA,GAAQ,IAAI,KAAA,CAAM,MAAA,CAAO,KAAK,CAAC;AAAA,OAChE,CAAA;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,kBAAA,CACN,OACA,KAAA,EACM;AACN,IAAA,IACE,KAAK,sBAAA,IACL,IAAA,CAAK,oBAAA,IACL,IAAA,CAAK,gBACL,CAAC,IAAA,CAAK,EAAA,IACN,CAAC,KAAK,EAAA,IACN,CAAC,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA,EACzB;AACA,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,IAAA,IAAA,CAAK,gBAAgB,UAAA,EAAW;AAChC,IAAA,IAAA,CAAK,SAAA,CAAU;AAAA,MACb,IAAA,EAAM,iBAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,cAAc,KAAA,EAAwC;AAC5D,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,CAAC,KAAK,EAAA,EAAI;AACxB,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IACE,UAAU,YAAA,IACV,KAAA,KAAU,cACV,KAAA,KAAU,SAAA,IACV,UAAU,QAAA,EACV;AACA,MAAA,OACE,KAAK,EAAA,CAAG,UAAA,KAAe,UACvB,IAAA,CAAK,EAAA,CAAG,oBAAoB,QAAA,IAC5B,IAAA,CAAK,EAAA,CAAG,eAAA,KAAoB,YAC5B,IAAA,CAAK,EAAA,CAAG,uBAAuB,QAAA,IAC/B,IAAA,CAAK,GAAG,kBAAA,KAAuB,cAAA;AAAA,IAEnC;AAEA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEQ,OAAA,GAAgB;AACtB,IAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AACpB,IAAA,IAAA,CAAK,8BAAA,EAA+B;AACpC,IAAA,IAAA,CAAK,uBAAuB,MAAA,EAAO;AACnC,IAAA,IAAA,CAAK,gBAAgB,OAAA,EAAQ;AAC7B,IAAA,IAAA,CAAK,0BAAA;AAAA,MACH,IAAI,MAAM,wDAAwD;AAAA,KACpE;AAEA,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,EAAA,EAAI;AACX,MAAA,IAAA,CAAK,GAAG,KAAA,EAAM;AACd,MAAA,IAAA,CAAK,EAAA,GAAK,IAAA;AAAA,IACZ;AAEA,IAAA,IAAI,KAAK,WAAA,EAAa;AACpB,MAAA,IAAA,CAAK,WAAA,CAAY,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAC5D,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAAA,IACrB;AAEA,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,IAAA,CAAK,aAAa,SAAA,GAAY,IAAA;AAC9B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AACnB,IAAA,IAAA,CAAK,oBAAA,GAAuB,KAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AACtB,IAAA,IAAA,CAAK,sBAAA,GAAyB,KAAA;AAC9B,IAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AACpB,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAAA,EAC7B;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAA,CAAK,qBAAA,EAAsB;AAC3B,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,cAAA,GAAiB,KAAA;AAAA,EACxB;AAAA,EAEQ,sBAAA,GAAyC;AAC/C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,IAAI,MAAM,iCAAiC,CAAA;AAAA,IACnD;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,IAAM,IAAA,CAAK,EAAA,CAAG,eAAe,MAAA,EAAQ;AAC7C,MAAA,MAAM,IAAI,MAAM,uBAAuB,CAAA;AAAA,IACzC;AAEA,IAAA,OAAO,IAAA,CAAK,EAAA;AAAA,EACd;AAAA,EAEA,MAAc,sBACZ,MAAA,EACkB;AAClB,IAAA,MAAM,KAAK,IAAA,CAAK,EAAA;AAChB,IAAA,IAAI,CAAC,EAAA,IAAM,IAAA,CAAK,uBAAA,EAAwB,EAAG;AACzC,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IAAI,EAAA,CAAG,uBAAuB,WAAA,EAAa;AACzC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAI,EAAA,CAAG,uBAAuB,cAAA,EAAgB;AAC5C,MAAA,MAAM,MAAM,IAAK,CAAA;AACjB,MAAA,IAAI,EAAA,CAAG,uBAAuB,cAAA,EAAgB;AAC5C,QAAA,OAAO,IAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,IACE,EAAA,CAAG,uBAAuB,QAAA,IAC1B,EAAA,CAAG,oBAAoB,QAAA,IACvB,EAAA,CAAG,oBAAoB,QAAA,EACvB;AACA,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,IAAI,OAAO,EAAA,CAAG,UAAA,KAAe,UAAA,EAAY;AACvC,MAAA,EAAA,CAAG,UAAA,EAAW;AAAA,IAChB;AAEA,IAAA,MAAM,QAAQ,MAAM,EAAA,CAAG,YAAY,EAAE,UAAA,EAAY,MAAM,CAAA;AACvD,IAAA,MAAM,EAAA,CAAG,oBAAoB,KAAK,CAAA;AAElC,IAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,mBAAA,CAAoB;AAAA,MAC/C,SAAA,EAAW,QAAA;AAAA,MACX,OAAA,EAAS,MAAA,IAAU,IAAA,CAAK,oBAAA,IAAwB,EAAC;AAAA,MACjD,UAAU,KAAA,CAAM;AAAA,KACjB,CAAA;AAED,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,sBAAA,CAAuB,SAAS,CAAA;AAEvD,IAAA,MAAM,GAAG,oBAAA,CAAqB;AAAA,MAC5B,IAAA,EAAM,QAAA;AAAA,MACN,GAAA,EAAK;AAAA,KACN,CAAA;AACD,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEQ,uBAAA,GAAmC;AACzC,IAAA,OACE,CAAC,KAAK,EAAA,IACN,CAAC,KAAK,EAAA,IACN,IAAA,CAAK,GAAG,UAAA,KAAe,QAAA,IACvB,KAAK,EAAA,CAAG,eAAA,KAAoB,YAC5B,IAAA,CAAK,EAAA,CAAG,oBAAoB,QAAA,IAC5B,IAAA,CAAK,GAAG,kBAAA,KAAuB,QAAA;AAAA,EAEnC;AAAA,EAEA,MAAc,kBACZ,MAAA,EACe;AACf,IAAA,IAAA,CAAK,sBAAA,GAAyB,IAAA;AAC9B,IAAA,IAAA,CAAK,OAAA,EAAQ;AACb,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,IAAU,IAAA,CAAK,oBAAoB,CAAA;AAAA,EACxD;AAAA,EAEQ,2BAAA,GAAoC;AAC1C,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,YAAA,CAAa,IAAA,CAAK,qBAAqB,SAAS,CAAA;AAChD,IAAA,IAAA,CAAK,qBAAqB,OAAA,EAAQ;AAClC,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAAA,EAC9B;AAAA,EAEQ,2BAA2B,KAAA,EAAoB;AACrD,IAAA,IAAI,CAAC,KAAK,oBAAA,EAAsB;AAC9B,MAAA;AAAA,IACF;AAEA,IAAA,YAAA,CAAa,IAAA,CAAK,qBAAqB,SAAS,CAAA;AAChD,IAAA,IAAA,CAAK,oBAAA,CAAqB,OAAO,KAAK,CAAA;AACtC,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAAA,EAC9B;AAAA,EAEQ,4BAA4B,KAAA,EAA6B;AAC/D,IAAA,MAAM,gBAAgB,IAAA,CAAK,oBAAA;AAC3B,IAAA,IAAI,CAAC,aAAA,EAAe;AAClB,MAAA,OAAO,KAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA,CAAM,KAAA,EAAO,QAAA,KAAa,aAAA,CAAc,OAAA;AAAA,EACjD;AAAA,EAEQ,yBAAA,GAAoC;AAC1C,IAAA,OACE,IAAA,CAAK,QAAQ,sBAAA,IAA0B,iCAAA;AAAA,EAE3C;AAAA,EAEQ,uBAAuB,SAAA,EAA6C;AAC1E,IAAA,IACE,SAAA,CAAU,YAAY,uBAAA,IACtB,SAAA,CAAU,cAAc,QAAA,IACxB,EAAE,eAAe,SAAA,CAAA,EACjB;AACA,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,wCAAwC,uBAAuB,CAAA,qBAAA,EAAwB,UAAU,OAAO,CAAA,CAAA,EAAI,UAAU,SAAS,CAAA;AAAA,OACjI;AAAA,IACF;AACA,IAAA,OAAO,SAAA,CAAU,SAAA;AAAA,EACnB;AAAA,EAEQ,qBAAA,GAA8B;AACpC,IAAA,IAAA,CAAK,aAAA,GAAgB,EAAA;AACrB,IAAA,IAAA,CAAK,2BAAA,GAA8B,KAAA;AACnC,IAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAAA,EAC/B;AAAA,EAEQ,8BAAA,GAAuC;AAC7C,IAAA,IAAI,CAAC,KAAK,2BAAA,EAA6B;AACrC,MAAA,IAAA,CAAK,2BAAA,GAA8B,IAAA;AACnC,MAAA,IAAA,CAAK,SAAA,CAAU,EAAE,IAAA,EAAM,4BAAA,EAA8B,CAAA;AAAA,IACvD;AAAA,EACF;AAAA,EAEQ,uBAAuB,IAAA,EAAoB;AACjD,IAAA,IAAA,CAAK,8BAAA,EAA+B;AAEpC,IAAA,IAAI,IAAA,KAAS,KAAK,aAAA,EAAe;AAC/B,MAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,UAAA,CAAW,IAAA,CAAK,aAAa,CAAA,GAC5C,IAAA,CAAK,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,MAAM,CAAA,GACpC,IAAA;AAEJ,MAAA,IAAA,CAAK,aAAA,GAAgB,IAAA;AACrB,MAAA,IAAI,KAAA,EAAO;AACT,QAAA,IAAA,CAAK,SAAA,CAAU;AAAA,UACb,IAAA,EAAM,sBAAA;AAAA,UACN,IAAA,EAAM;AAAA,SACP,CAAA;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,KAAA,EAAqC;AACrD,IAAA,KAAA,MAAW,QAAA,IAAY,KAAK,cAAA,EAAgB;AAC1C,MAAA,QAAA,CAAS,KAAK,CAAA;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,OAAO,IAAA,EAAuB;AACpC,IAAA,IAAI,IAAA,CAAK,QAAQ,KAAA,EAAO;AACtB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAG,IAAI,CAAA;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,oBAAoB,SAAA,EAA4B;AACtD,IAAA,IAAI,qBAAA,CAAsB,GAAA,CAAI,SAAS,CAAA,EAAG;AACxC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,SAAA,CAAU,WAAW,uBAAuB,CAAA;AAAA,EACrD;AACF;AAEO,SAAS,2BACd,OAAA,EACyB;AACzB,EAAA,OAAO,IAAI,qBAAqB,OAAO,CAAA;AACzC;AAEA,SAAS,8BACP,MAAA,EACyB;AACzB,EAAA,MAAM,KAAA,GAAiC;AAAA,IACrC,MAAA,EAAQ;AAAA,MACN,KAAA,EAAO,QAAQ,KAAA,IAAS;AAAA;AAC1B,GACF;AAEA,EAAA,MAAM,gBAAgB,MAAA,EAAQ,uBAAA;AAC9B,EAAA,IAAI,kBAAkB,MAAA,EAAW;AAC/B,IAAA,IAAI,aAAA,CAAc,YAAY,KAAA,EAAO;AACnC,MAAA,KAAA,CAAM,KAAA,GAAQ,EAAE,aAAA,EAAe,IAAA,EAAK;AAAA,IACtC,CAAA,MAAA,IAAW,aAAA,CAAc,KAAA,KAAU,MAAA,EAAW;AAC5C,MAAA,KAAA,CAAM,QAAQ,EAAE,aAAA,EAAe,EAAE,KAAA,EAAO,aAAA,CAAc,OAAM,EAAE;AAAA,IAChE;AAAA,EACF;AAEA,EAAA,MAAM,OAAA,GAAmC;AAAA,IACvC,KAAA;AAAA,IACA,WAAA,EAAa,QAAQ,UAAA,IAAc;AAAA,GACrC;AAEA,EAAA,IAAI,QAAQ,YAAA,EAAc;AACxB,IAAA,OAAA,CAAQ,eAAe,MAAA,CAAO,YAAA;AAAA,EAChC;AAEA,EAAA,IAAI,MAAA,EAAQ,gBAAgB,MAAA,EAAW;AACrC,IAAA,OAAA,CAAQ,cAAc,MAAA,CAAO,WAAA;AAAA,EAC/B;AAEA,EAAA,IAAI,MAAA,EAAQ,cAAc,MAAA,EAAW;AACnC,IAAA,OAAA,CAAQ,6BAA6B,MAAA,CAAO,SAAA;AAAA,EAC9C;AAEA,EAAA,IAAI,MAAA,IAAU,WAAW,MAAA,EAAQ;AAC/B,IAAA,OAAA,CAAQ,KAAA,GAAQ,MAAA,CAAO,KAAA,IAAS,EAAC;AAAA,EACnC;AAEA,EAAA,OAAO,OAAA;AACT","file":"chunk-MI4QTRWT.mjs","sourcesContent":["import type {\n RealtimeReconnectCause,\n RealtimeSessionBootstrap,\n RealtimeSessionConfig,\n RealtimeSessionRequest,\n} from \"@charivo/core\";\nimport { OPENAI_REALTIME_ADAPTER } from \"@charivo/core\";\nimport { acquireMicrophoneStream } from \"../internal/microphone\";\nimport {\n bindTransportLifecycle,\n createIceDisconnectDebouncer,\n DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS,\n replaceMicrophoneTrack,\n} from \"../internal/webrtc-lifecycle\";\nimport {\n DEFAULT_REQUEST_TIMEOUT_MS,\n fetchWithTimeout,\n isRealtimeSessionBootstrap,\n isRecord,\n} from \"../internal/shared\";\nimport { delay } from \"../internal/timing\";\nimport { LipSyncAnalyzer } from \"../openai-agents/lip-sync-analyzer\";\nimport type { RealtimeTransportClient, RealtimeTransportEvent } from \"../types\";\nimport { DEFAULT_OPENAI_REALTIME_VOICE } from \"./defaults\";\n\ninterface ServerError {\n code?: string;\n event_id?: string;\n message?: string;\n}\n\ninterface ServerEventItem {\n call_id?: string;\n name?: string;\n arguments?: string;\n}\n\ninterface ServerEvent {\n type: string;\n event_id?: string;\n delta?: string;\n text?: string;\n transcript?: string;\n call_id?: string;\n item_id?: string;\n name?: string;\n arguments?: string;\n item?: ServerEventItem;\n error?: ServerError;\n response?: {\n id?: string;\n model?: string;\n usage?: Record<string, unknown>;\n };\n}\n\nexport interface OpenAIRealtimeClientOptions {\n apiEndpoint?: string;\n debug?: boolean;\n sessionUpdateTimeoutMs?: number;\n sessionBootstrap?: (\n request: RealtimeSessionRequest,\n ) => Promise<RealtimeSessionBootstrap>;\n}\n\ninterface PendingSessionUpdate {\n eventId: string;\n resolve: () => void;\n reject: (error: Error) => void;\n timeoutId: ReturnType<typeof setTimeout>;\n}\n\nconst DEBUG_EVENT_ALLOWLIST = new Set<string>([\n \"session.created\",\n \"session.updated\",\n \"response.created\",\n \"response.done\",\n \"response.function_call_arguments.done\",\n \"conversation.item.input_audio_transcription.completed\",\n \"error\",\n]);\nconst DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5_000;\n\n/**\n * OpenAI-specific realtime transport client.\n *\n * This package normalizes OpenAI Realtime WebRTC events into the transport\n * event contract defined by `@charivo/realtime`.\n */\nexport class OpenAIRealtimeClient implements RealtimeTransportClient {\n private pc: RTCPeerConnection | null = null;\n private dc: RTCDataChannel | null = null;\n private audioElement: HTMLAudioElement | null = null;\n private mediaStream: MediaStream | null = null;\n private audioSender: RTCRtpSender | null = null;\n private readonly lipSyncAnalyzer = new LipSyncAnalyzer({\n onRms: (rms) => this.emitEvent({ type: \"audio.lipsync\", rms }),\n onError: (error) => console.error(\"Failed to setup audio analysis:\", error),\n });\n private isResponseInProgress = false;\n private cancelInFlight = false;\n private hasStartedAssistantResponse = false;\n private hasStartedAudioOutput = false;\n private assistantText = \"\";\n private nextSessionUpdateId = 1;\n private pendingSessionUpdate: PendingSessionUpdate | null = null;\n private currentSessionConfig?: RealtimeSessionConfig;\n private connectionLossNotified = false;\n private isExplicitDisconnect = false;\n private isRecovering = false;\n private isCleaningUp = false;\n private teardownTransportLifecycle?: () => void;\n private readonly iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {\n this.emitConnectionLost(\"ice-disconnected\");\n }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);\n private eventCallbacks = new Set<(event: RealtimeTransportEvent) => void>();\n\n constructor(private options: OpenAIRealtimeClientOptions = {}) {}\n\n async connect(config?: RealtimeSessionConfig): Promise<void> {\n try {\n if (this.isExplicitDisconnect && this.isRecovering) {\n throw new Error(\"Realtime session disconnect requested\");\n }\n\n this.log(\"Starting OpenAI Realtime WebRTC connection\");\n\n this.isExplicitDisconnect = false;\n this.isRecovering = false;\n this.connectionLossNotified = false;\n this.currentSessionConfig = config;\n this.pc = new RTCPeerConnection();\n this.audioElement = document.createElement(\"audio\");\n this.audioElement.autoplay = true;\n this.audioElement.setAttribute(\"playsinline\", \"true\");\n void this.lipSyncAnalyzer.prepareAudioContext().catch(() => undefined);\n this.bindConnectionEvents();\n this.bindTransportLifecycleEvents();\n\n this.pc.ontrack = (event) => {\n if (this.audioElement) {\n this.audioElement.srcObject = event.streams[0];\n }\n this.lipSyncAnalyzer.attachStream(event.streams[0]);\n };\n\n try {\n this.mediaStream = await acquireMicrophoneStream();\n const audioTrack = this.mediaStream.getTracks()[0];\n this.audioSender =\n this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;\n } catch {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n this.dc = this.pc.createDataChannel(\"oai-events\");\n this.dc.onmessage = (event) => {\n const payload = JSON.parse(event.data) as ServerEvent;\n this.handleServerEvent(payload);\n };\n this.dc.onerror = () => {\n this.emitEvent({\n type: \"error\",\n error: new Error(\"DataChannel error\"),\n });\n };\n\n const offer = await this.pc.createOffer();\n await this.pc.setLocalDescription(offer);\n\n const bootstrap = await this.getSessionBootstrap({\n transport: \"webrtc\",\n session: config ?? {},\n sdpOffer: offer.sdp,\n });\n\n const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);\n\n await this.pc.setRemoteDescription({\n type: \"answer\",\n sdp: answerSdp,\n });\n\n this.emitEvent({ type: \"session.started\" });\n } catch (error) {\n this.cleanup();\n throw error;\n }\n }\n\n async disconnect(): Promise<void> {\n this.log(\"Disconnecting OpenAI Realtime WebRTC\");\n this.isExplicitDisconnect = true;\n this.isRecovering = false;\n this.cleanup();\n }\n\n async updateSession(config?: RealtimeSessionConfig): Promise<void> {\n if (this.isResponseInProgress) {\n throw new Error(\n \"Cannot update the realtime session while a response is in progress. Call interrupt() first.\",\n );\n }\n\n const dc = this.requireOpenDataChannel();\n const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;\n\n if (this.pendingSessionUpdate) {\n throw new Error(\"Realtime session update already in progress\");\n }\n\n dc.send(\n JSON.stringify({\n type: \"session.update\",\n event_id: eventId,\n session: toOpenAIRealtimeSessionUpdate(config),\n }),\n );\n\n await new Promise<void>((resolve, reject) => {\n const timeoutId = setTimeout(() => {\n this.rejectPendingSessionUpdate(\n new Error(\n `Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`,\n ),\n );\n }, this.getSessionUpdateTimeoutMs());\n\n this.pendingSessionUpdate = {\n eventId,\n resolve,\n reject,\n timeoutId,\n };\n });\n\n this.currentSessionConfig = config;\n }\n\n async recover(config?: RealtimeSessionConfig): Promise<void> {\n this.isRecovering = true;\n this.connectionLossNotified = true;\n this.currentSessionConfig = config;\n\n try {\n if (await this.tryIceRestartRecovery(config)) {\n this.connectionLossNotified = false;\n return;\n }\n\n await this.rebuildConnection(config);\n this.connectionLossNotified = false;\n } finally {\n this.isRecovering = false;\n }\n }\n\n async sendText(text: string): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n if (this.isResponseInProgress) {\n console.warn(\"⚠️ Response already in progress, skipping request\");\n return;\n }\n\n this.dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text,\n },\n ],\n },\n }),\n );\n this.dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async sendAudio(_audio: ArrayBuffer): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n console.warn(\n \"sendAudio is not needed with WebRTC - audio is automatically transmitted\",\n );\n }\n\n async sendToolResult(\n callId: string,\n output: Record<string, unknown>,\n ): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n const dc = this.requireOpenDataChannel();\n\n dc.send(\n JSON.stringify({\n type: \"conversation.item.create\",\n item: {\n type: \"function_call_output\",\n call_id: callId,\n output: JSON.stringify(output),\n },\n }),\n );\n dc.send(JSON.stringify({ type: \"response.create\" }));\n\n this.beginResponseRequest();\n }\n\n async interrupt(): Promise<void> {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n const dc = this.requireOpenDataChannel();\n\n if (!this.isResponseInProgress) {\n return;\n }\n\n dc.send(JSON.stringify({ type: \"response.cancel\" }));\n this.isResponseInProgress = false;\n this.cancelInFlight = true;\n }\n\n onEvent(callback: (event: RealtimeTransportEvent) => void): void {\n this.eventCallbacks.add(callback);\n }\n\n private handleServerEvent(event: ServerEvent): void {\n if (this.options.debug && this.shouldLogDebugEvent(event.type)) {\n this.log(\"📡 [OpenAI Realtime Event]\", event.type, event);\n }\n\n switch (event.type) {\n case \"session.created\":\n return;\n\n case \"session.updated\":\n this.resolvePendingSessionUpdate();\n return;\n\n case \"response.audio.delta\":\n case \"output_audio_buffer.started\":\n if (this.cancelInFlight) {\n return;\n }\n\n if (!this.hasStartedAudioOutput) {\n this.hasStartedAudioOutput = true;\n this.emitEvent({ type: \"audio.output.started\" });\n }\n return;\n\n case \"response.audio.done\":\n case \"response.output_audio.done\":\n case \"output_audio_buffer.stopped\":\n if (this.cancelInFlight) {\n this.hasStartedAudioOutput = false;\n return;\n }\n\n this.emitEvent({ type: \"audio.output.ended\" });\n this.hasStartedAudioOutput = false;\n return;\n\n case \"response.done\":\n this.isResponseInProgress = false;\n if (this.cancelInFlight) {\n this.cancelInFlight = false;\n this.resetResponseTracking();\n return;\n }\n\n // Tool-using turns fire response.done twice per user message: once\n // after the tool call (no text this cycle) and once after the\n // follow-up reply. Skip the first one so consumers see a single\n // completion per user turn. Keeping tracking live means the\n // follow-up cycle does not re-emit assistant.response.started.\n if (!this.assistantText) {\n return;\n }\n\n this.emitEvent({\n type: \"assistant.response.completed\",\n text: this.assistantText,\n usage: event.response?.usage,\n model: event.response?.model,\n responseId: event.response?.id,\n });\n this.resetResponseTracking();\n return;\n\n case \"response.audio_transcript.delta\":\n case \"response.output_audio_transcript.delta\":\n case \"response.output_text.delta\":\n if (this.cancelInFlight || !event.delta) {\n return;\n }\n\n this.ensureAssistantResponseStarted();\n\n this.assistantText += event.delta;\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: event.delta,\n });\n return;\n\n case \"response.output_text.done\":\n if (this.cancelInFlight || !event.text) {\n return;\n }\n this.emitFinalAssistantText(event.text);\n return;\n\n case \"response.output_audio_transcript.done\":\n if (this.cancelInFlight || !event.transcript) {\n return;\n }\n this.emitFinalAssistantText(event.transcript);\n return;\n\n case \"conversation.item.input_audio_transcription.completed\":\n if (event.transcript) {\n this.emitEvent({\n type: \"user.transcript\",\n text: event.transcript,\n });\n }\n return;\n\n case \"response.function_call_arguments.done\":\n this.handleToolCallDone(event);\n return;\n\n case \"error\":\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n {\n const error = new Error(event.error?.message || \"Unknown error\");\n if (this.isPendingSessionUpdateError(event)) {\n this.rejectPendingSessionUpdate(error);\n }\n\n this.emitEvent({\n type: \"error\",\n error,\n });\n }\n return;\n\n default:\n return;\n }\n }\n\n private handleToolCallDone(event: ServerEvent): void {\n const callId = event.call_id || event.item?.call_id || event.item_id;\n const name = event.name || event.item?.name;\n const argsJson = event.arguments || event.item?.arguments;\n\n if (!name || !argsJson) {\n console.warn(\"⚠️ Tool call done but missing name or arguments\");\n return;\n }\n\n let args: Record<string, unknown> = {};\n try {\n const parsed = JSON.parse(argsJson) as unknown;\n if (isRecord(parsed)) {\n args = parsed;\n }\n } catch (error) {\n console.error(\"Failed to parse tool call args\", error, argsJson);\n return;\n }\n\n this.emitEvent({\n type: \"tool.call\",\n name,\n args,\n callId,\n });\n }\n\n private async getSessionBootstrap(\n request: RealtimeSessionRequest,\n ): Promise<RealtimeSessionBootstrap> {\n if (this.options.sessionBootstrap) {\n return this.options.sessionBootstrap(request);\n }\n\n const apiEndpoint = this.options.apiEndpoint;\n if (!apiEndpoint) {\n throw new Error(\n \"OpenAI realtime client requires apiEndpoint or sessionBootstrap\",\n );\n }\n\n const response = await fetchWithTimeout(\n apiEndpoint,\n {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(request),\n },\n `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,\n );\n\n if (!response.ok) {\n const errorText = await response.text();\n throw new Error(`Failed to create Realtime session: ${errorText}`);\n }\n\n const bootstrap = (await response.json()) as unknown;\n if (!isRealtimeSessionBootstrap(bootstrap)) {\n throw new Error(\"Invalid realtime session bootstrap response\");\n }\n\n return bootstrap as RealtimeSessionBootstrap;\n }\n\n private bindConnectionEvents(): void {\n if (!this.pc) {\n return;\n }\n\n this.pc.addEventListener(\"iceconnectionstatechange\", () => {\n const iceState = this.pc?.iceConnectionState;\n if (iceState === \"failed\") {\n this.iceDisconnectDebouncer.cancel();\n this.emitConnectionLost(\"ice-failed\");\n } else if (iceState === \"disconnected\") {\n this.iceDisconnectDebouncer.schedule();\n } else {\n this.iceDisconnectDebouncer.cancel();\n }\n });\n\n this.pc.addEventListener(\"connectionstatechange\", () => {\n const connectionState = this.pc?.connectionState;\n if (connectionState === \"failed\" || connectionState === \"closed\") {\n this.iceDisconnectDebouncer.cancel();\n this.emitConnectionLost(\"connection-failed\");\n }\n });\n }\n\n private bindTransportLifecycleEvents(): void {\n if (this.teardownTransportLifecycle) {\n return;\n }\n\n this.teardownTransportLifecycle = bindTransportLifecycle({\n onHidden: this.handleHidden,\n onOnline: this.handleOnline,\n onPageHide: this.handlePageHide,\n onPageShow: this.handlePageShow,\n onVisible: this.handleVisible,\n onDeviceChange: this.handleDeviceChange,\n });\n }\n\n private unbindTransportLifecycleEvents(): void {\n this.teardownTransportLifecycle?.();\n this.teardownTransportLifecycle = undefined;\n }\n\n private readonly handleOnline = (): void => {\n this.emitConnectionLost(\"online\");\n };\n\n private readonly handleHidden = (): void => {\n this.lipSyncAnalyzer.pause();\n };\n\n private readonly handleVisible = (): void => {\n this.lipSyncAnalyzer.resume();\n this.emitConnectionLost(\"visibility\");\n };\n\n private readonly handlePageHide = (): void => {\n this.lipSyncAnalyzer.pause();\n };\n\n private readonly handlePageShow = (event: PageTransitionEvent): void => {\n this.lipSyncAnalyzer.resume();\n if (event.persisted) {\n this.emitConnectionLost(\"pageshow\");\n }\n };\n\n private readonly handleDeviceChange = (): void => {\n void this.refreshMicrophoneTrack();\n };\n\n private async refreshMicrophoneTrack(): Promise<void> {\n if (!this.pc || this.isExplicitDisconnect) {\n return;\n }\n\n try {\n const { audioSender, mediaStream } = await replaceMicrophoneTrack({\n audioSender: this.audioSender,\n mediaStream: this.mediaStream,\n peerConnection: this.pc,\n });\n this.audioSender = audioSender;\n this.mediaStream = mediaStream;\n } catch (error) {\n this.emitEvent({\n type: \"error\",\n error: error instanceof Error ? error : new Error(String(error)),\n });\n }\n }\n\n private emitConnectionLost(\n cause: RealtimeReconnectCause,\n error?: Error,\n ): void {\n if (\n this.connectionLossNotified ||\n this.isExplicitDisconnect ||\n this.isCleaningUp ||\n !this.pc ||\n !this.dc ||\n !this.needsRecovery(cause)\n ) {\n return;\n }\n\n this.connectionLossNotified = true;\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.resetResponseTracking();\n this.lipSyncAnalyzer.stopOutput();\n this.emitEvent({\n type: \"connection.lost\",\n cause,\n error,\n });\n }\n\n private needsRecovery(cause: RealtimeReconnectCause): boolean {\n if (!this.pc || !this.dc) {\n return false;\n }\n\n if (\n cause === \"visibility\" ||\n cause === \"pageshow\" ||\n cause === \"offline\" ||\n cause === \"online\"\n ) {\n return (\n this.dc.readyState !== \"open\" ||\n this.pc.connectionState === \"failed\" ||\n this.pc.connectionState === \"closed\" ||\n this.pc.iceConnectionState === \"failed\" ||\n this.pc.iceConnectionState === \"disconnected\"\n );\n }\n\n return true;\n }\n\n private cleanup(): void {\n this.isCleaningUp = true;\n this.unbindTransportLifecycleEvents();\n this.iceDisconnectDebouncer.cancel();\n this.lipSyncAnalyzer.cleanup();\n this.rejectPendingSessionUpdate(\n new Error(\"Realtime session ended before session update completed\"),\n );\n\n if (this.dc) {\n this.dc.close();\n this.dc = null;\n }\n\n if (this.pc) {\n this.pc.close();\n this.pc = null;\n }\n\n if (this.mediaStream) {\n this.mediaStream.getTracks().forEach((track) => track.stop());\n this.mediaStream = null;\n }\n\n if (this.audioElement) {\n this.audioElement.srcObject = null;\n this.audioElement = null;\n }\n\n this.audioSender = null;\n this.isResponseInProgress = false;\n this.cancelInFlight = false;\n this.connectionLossNotified = false;\n this.isCleaningUp = false;\n this.resetResponseTracking();\n }\n\n private beginResponseRequest(): void {\n this.resetResponseTracking();\n this.isResponseInProgress = true;\n this.cancelInFlight = false;\n }\n\n private requireOpenDataChannel(): RTCDataChannel {\n if (this.isRecovering) {\n throw new Error(\"Realtime transport reconnecting\");\n }\n\n if (!this.dc || this.dc.readyState !== \"open\") {\n throw new Error(\"DataChannel not ready\");\n }\n\n return this.dc;\n }\n\n private async tryIceRestartRecovery(\n config?: RealtimeSessionConfig,\n ): Promise<boolean> {\n const pc = this.pc;\n if (!pc || this.shouldRebuildConnection()) {\n return false;\n }\n\n if (pc.iceConnectionState === \"connected\") {\n return true;\n }\n\n if (pc.iceConnectionState === \"disconnected\") {\n await delay(1_500);\n if (pc.iceConnectionState !== \"disconnected\") {\n return true;\n }\n }\n\n if (\n pc.iceConnectionState === \"failed\" ||\n pc.connectionState === \"failed\" ||\n pc.connectionState === \"closed\"\n ) {\n return false;\n }\n\n if (typeof pc.restartIce === \"function\") {\n pc.restartIce();\n }\n\n const offer = await pc.createOffer({ iceRestart: true });\n await pc.setLocalDescription(offer);\n\n const bootstrap = await this.getSessionBootstrap({\n transport: \"webrtc\",\n session: config ?? this.currentSessionConfig ?? {},\n sdpOffer: offer.sdp,\n });\n\n const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);\n\n await pc.setRemoteDescription({\n type: \"answer\",\n sdp: answerSdp,\n });\n return true;\n }\n\n private shouldRebuildConnection(): boolean {\n return (\n !this.pc ||\n !this.dc ||\n this.dc.readyState === \"closed\" ||\n this.pc.connectionState === \"failed\" ||\n this.pc.connectionState === \"closed\" ||\n this.pc.iceConnectionState === \"failed\"\n );\n }\n\n private async rebuildConnection(\n config?: RealtimeSessionConfig,\n ): Promise<void> {\n this.connectionLossNotified = true;\n this.cleanup();\n await this.connect(config ?? this.currentSessionConfig);\n }\n\n private resolvePendingSessionUpdate(): void {\n if (!this.pendingSessionUpdate) {\n return;\n }\n\n clearTimeout(this.pendingSessionUpdate.timeoutId);\n this.pendingSessionUpdate.resolve();\n this.pendingSessionUpdate = null;\n }\n\n private rejectPendingSessionUpdate(error: Error): void {\n if (!this.pendingSessionUpdate) {\n return;\n }\n\n clearTimeout(this.pendingSessionUpdate.timeoutId);\n this.pendingSessionUpdate.reject(error);\n this.pendingSessionUpdate = null;\n }\n\n private isPendingSessionUpdateError(event: ServerEvent): boolean {\n const pendingUpdate = this.pendingSessionUpdate;\n if (!pendingUpdate) {\n return false;\n }\n\n return event.error?.event_id === pendingUpdate.eventId;\n }\n\n private getSessionUpdateTimeoutMs(): number {\n return (\n this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS\n );\n }\n\n private resolveWebRTCAnswerSdp(bootstrap: RealtimeSessionBootstrap): string {\n if (\n bootstrap.adapter !== OPENAI_REALTIME_ADAPTER ||\n bootstrap.transport !== \"webrtc\" ||\n !(\"answerSdp\" in bootstrap)\n ) {\n throw new Error(\n `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`,\n );\n }\n return bootstrap.answerSdp;\n }\n\n private resetResponseTracking(): void {\n this.assistantText = \"\";\n this.hasStartedAssistantResponse = false;\n this.hasStartedAudioOutput = false;\n }\n\n private ensureAssistantResponseStarted(): void {\n if (!this.hasStartedAssistantResponse) {\n this.hasStartedAssistantResponse = true;\n this.emitEvent({ type: \"assistant.response.started\" });\n }\n }\n\n private emitFinalAssistantText(full: string): void {\n this.ensureAssistantResponseStarted();\n\n if (full !== this.assistantText) {\n const delta = full.startsWith(this.assistantText)\n ? full.slice(this.assistantText.length)\n : full;\n\n this.assistantText = full;\n if (delta) {\n this.emitEvent({\n type: \"assistant.text.delta\",\n text: delta,\n });\n }\n }\n }\n\n private emitEvent(event: RealtimeTransportEvent): void {\n for (const callback of this.eventCallbacks) {\n callback(event);\n }\n }\n\n private log(...args: unknown[]): void {\n if (this.options.debug) {\n console.log(...args);\n }\n }\n\n private shouldLogDebugEvent(eventType: string): boolean {\n if (DEBUG_EVENT_ALLOWLIST.has(eventType)) {\n return true;\n }\n\n return eventType.startsWith(\"response.output_text.\");\n }\n}\n\nexport function createOpenAIRealtimeClient(\n options?: OpenAIRealtimeClientOptions,\n): RealtimeTransportClient {\n return new OpenAIRealtimeClient(options);\n}\n\nfunction toOpenAIRealtimeSessionUpdate(\n config?: RealtimeSessionConfig,\n): Record<string, unknown> {\n const audio: Record<string, unknown> = {\n output: {\n voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE,\n },\n };\n\n const transcription = config?.inputAudioTranscription;\n if (transcription !== undefined) {\n if (transcription.enabled === false) {\n audio.input = { transcription: null };\n } else if (transcription.model !== undefined) {\n audio.input = { transcription: { model: transcription.model } };\n }\n }\n\n const session: Record<string, unknown> = {\n audio,\n tool_choice: config?.toolChoice ?? \"auto\",\n };\n\n if (config?.instructions) {\n session.instructions = config.instructions;\n }\n\n if (config?.temperature !== undefined) {\n session.temperature = config.temperature;\n }\n\n if (config?.maxTokens !== undefined) {\n session.max_response_output_tokens = config.maxTokens;\n }\n\n if (config && \"tools\" in config) {\n session.tools = config.tools ?? [];\n }\n\n return session;\n}\n"]}
|