@charivo/realtime 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -7
- package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
- package/dist/chunk-DHNRDHST.mjs.map +1 -0
- package/dist/chunk-PDODY2T7.mjs +65 -0
- package/dist/chunk-PDODY2T7.mjs.map +1 -0
- package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
- package/dist/chunk-PTFZ7CML.mjs.map +1 -0
- package/dist/chunk-Y7JO2FFX.mjs +121 -0
- package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
- package/dist/index.d.mts +16 -4
- package/dist/index.d.ts +16 -4
- package/dist/index.js +240 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +238 -98
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +37 -1
- package/dist/openai/index.d.ts +37 -1
- package/dist/openai/index.js +427 -14
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.d.mts +26 -1
- package/dist/openai-agents/index.d.ts +26 -1
- package/dist/openai-agents/index.js +311 -18
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -2
- package/dist/remote/index.d.mts +3 -1
- package/dist/remote/index.d.ts +3 -1
- package/dist/remote/index.js +663 -32
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +10 -3
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
- package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
- package/package.json +3 -3
- package/dist/chunk-IIKECRXN.mjs.map +0 -1
- package/dist/chunk-JGJZKKMH.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/openai/index.js
CHANGED
|
@@ -4,6 +4,87 @@ var core = require('@charivo/core');
|
|
|
4
4
|
|
|
5
5
|
// src/openai/client.ts
|
|
6
6
|
|
|
7
|
+
// src/instructions.ts
|
|
8
|
+
var DEFAULT_REALTIME_VOICE = "marin";
|
|
9
|
+
|
|
10
|
+
// src/internal/microphone.ts
|
|
11
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
12
|
+
echoCancellation: true,
|
|
13
|
+
noiseSuppression: true,
|
|
14
|
+
autoGainControl: true
|
|
15
|
+
};
|
|
16
|
+
async function acquireMicrophoneStream() {
|
|
17
|
+
return navigator.mediaDevices.getUserMedia({
|
|
18
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
22
|
+
function bindTransportLifecycle(callbacks) {
|
|
23
|
+
const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
24
|
+
onHidden: callbacks.onHidden,
|
|
25
|
+
onOnline: callbacks.onOnline,
|
|
26
|
+
onPageHide: callbacks.onPageHide,
|
|
27
|
+
onPageShow: callbacks.onPageShow,
|
|
28
|
+
onVisible: callbacks.onVisible
|
|
29
|
+
});
|
|
30
|
+
if (callbacks.onDeviceChange) {
|
|
31
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
32
|
+
"devicechange",
|
|
33
|
+
callbacks.onDeviceChange
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
return () => {
|
|
37
|
+
teardownBrowserLifecycle();
|
|
38
|
+
if (callbacks.onDeviceChange) {
|
|
39
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
40
|
+
"devicechange",
|
|
41
|
+
callbacks.onDeviceChange
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
async function replaceMicrophoneTrack(state) {
|
|
47
|
+
if (!state.peerConnection) {
|
|
48
|
+
throw new Error("Realtime transport not active");
|
|
49
|
+
}
|
|
50
|
+
const nextStream = await acquireMicrophoneStream();
|
|
51
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
52
|
+
if (!nextTrack) {
|
|
53
|
+
throw new Error("Microphone access required for Realtime API");
|
|
54
|
+
}
|
|
55
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
56
|
+
if (!sender) {
|
|
57
|
+
throw new Error("No outbound audio sender available");
|
|
58
|
+
}
|
|
59
|
+
await sender.replaceTrack(nextTrack);
|
|
60
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
61
|
+
return {
|
|
62
|
+
audioSender: sender,
|
|
63
|
+
mediaStream: nextStream
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
67
|
+
let pending = null;
|
|
68
|
+
return {
|
|
69
|
+
schedule() {
|
|
70
|
+
if (pending) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
pending = setTimeout(() => {
|
|
74
|
+
pending = null;
|
|
75
|
+
onDisconnect();
|
|
76
|
+
}, delayMs);
|
|
77
|
+
},
|
|
78
|
+
cancel() {
|
|
79
|
+
if (!pending) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
clearTimeout(pending);
|
|
83
|
+
pending = null;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
7
88
|
// src/internal/shared.ts
|
|
8
89
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
9
90
|
function isRecord(value) {
|
|
@@ -52,6 +133,7 @@ var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
|
52
133
|
"conversation.item.input_audio_transcription.completed",
|
|
53
134
|
"error"
|
|
54
135
|
]);
|
|
136
|
+
var DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5e3;
|
|
55
137
|
var OpenAIRealtimeClient = class {
|
|
56
138
|
constructor(options = {}) {
|
|
57
139
|
this.options = options;
|
|
@@ -60,7 +142,9 @@ var OpenAIRealtimeClient = class {
|
|
|
60
142
|
dc = null;
|
|
61
143
|
audioElement = null;
|
|
62
144
|
mediaStream = null;
|
|
145
|
+
audioSender = null;
|
|
63
146
|
audioContext = null;
|
|
147
|
+
audioSource = null;
|
|
64
148
|
analyser = null;
|
|
65
149
|
lipSyncInterval = null;
|
|
66
150
|
isResponseInProgress = false;
|
|
@@ -68,13 +152,35 @@ var OpenAIRealtimeClient = class {
|
|
|
68
152
|
hasStartedAssistantResponse = false;
|
|
69
153
|
hasStartedAudioOutput = false;
|
|
70
154
|
assistantText = "";
|
|
155
|
+
nextSessionUpdateId = 1;
|
|
156
|
+
pendingSessionUpdate = null;
|
|
157
|
+
currentSessionConfig;
|
|
158
|
+
connectionLossNotified = false;
|
|
159
|
+
isExplicitDisconnect = false;
|
|
160
|
+
isRecovering = false;
|
|
161
|
+
isCleaningUp = false;
|
|
162
|
+
teardownTransportLifecycle;
|
|
163
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
164
|
+
this.emitConnectionLost("ice-disconnected");
|
|
165
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
71
166
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
72
167
|
async connect(config) {
|
|
73
168
|
try {
|
|
169
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
170
|
+
throw new Error("Realtime session disconnect requested");
|
|
171
|
+
}
|
|
74
172
|
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
173
|
+
this.isExplicitDisconnect = false;
|
|
174
|
+
this.isRecovering = false;
|
|
175
|
+
this.connectionLossNotified = false;
|
|
176
|
+
this.currentSessionConfig = config;
|
|
75
177
|
this.pc = new RTCPeerConnection();
|
|
76
178
|
this.audioElement = document.createElement("audio");
|
|
77
179
|
this.audioElement.autoplay = true;
|
|
180
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
181
|
+
this.prepareAudioAnalysis();
|
|
182
|
+
this.bindConnectionEvents();
|
|
183
|
+
this.bindTransportLifecycleEvents();
|
|
78
184
|
this.pc.ontrack = (event) => {
|
|
79
185
|
if (this.audioElement) {
|
|
80
186
|
this.audioElement.srcObject = event.streams[0];
|
|
@@ -82,11 +188,9 @@ var OpenAIRealtimeClient = class {
|
|
|
82
188
|
this.setupAudioAnalysis(event.streams[0]);
|
|
83
189
|
};
|
|
84
190
|
try {
|
|
85
|
-
this.mediaStream = await
|
|
86
|
-
audio: true
|
|
87
|
-
});
|
|
191
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
88
192
|
const audioTrack = this.mediaStream.getTracks()[0];
|
|
89
|
-
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
193
|
+
this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
|
|
90
194
|
} catch {
|
|
91
195
|
throw new Error("Microphone access required for Realtime API");
|
|
92
196
|
}
|
|
@@ -125,9 +229,64 @@ var OpenAIRealtimeClient = class {
|
|
|
125
229
|
}
|
|
126
230
|
async disconnect() {
|
|
127
231
|
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
232
|
+
this.isExplicitDisconnect = true;
|
|
233
|
+
this.isRecovering = false;
|
|
128
234
|
this.cleanup();
|
|
129
235
|
}
|
|
236
|
+
async updateSession(config) {
|
|
237
|
+
if (this.isResponseInProgress) {
|
|
238
|
+
throw new Error(
|
|
239
|
+
"Cannot update the realtime session while a response is in progress. Call interrupt() first."
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
const dc = this.requireOpenDataChannel();
|
|
243
|
+
const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;
|
|
244
|
+
if (this.pendingSessionUpdate) {
|
|
245
|
+
throw new Error("Realtime session update already in progress");
|
|
246
|
+
}
|
|
247
|
+
dc.send(
|
|
248
|
+
JSON.stringify({
|
|
249
|
+
type: "session.update",
|
|
250
|
+
event_id: eventId,
|
|
251
|
+
session: toOpenAIRealtimeSessionUpdate(config)
|
|
252
|
+
})
|
|
253
|
+
);
|
|
254
|
+
await new Promise((resolve, reject) => {
|
|
255
|
+
const timeoutId = setTimeout(() => {
|
|
256
|
+
this.rejectPendingSessionUpdate(
|
|
257
|
+
new Error(
|
|
258
|
+
`Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`
|
|
259
|
+
)
|
|
260
|
+
);
|
|
261
|
+
}, this.getSessionUpdateTimeoutMs());
|
|
262
|
+
this.pendingSessionUpdate = {
|
|
263
|
+
eventId,
|
|
264
|
+
resolve,
|
|
265
|
+
reject,
|
|
266
|
+
timeoutId
|
|
267
|
+
};
|
|
268
|
+
});
|
|
269
|
+
this.currentSessionConfig = config;
|
|
270
|
+
}
|
|
271
|
+
async recover(config) {
|
|
272
|
+
this.isRecovering = true;
|
|
273
|
+
this.connectionLossNotified = true;
|
|
274
|
+
this.currentSessionConfig = config;
|
|
275
|
+
try {
|
|
276
|
+
if (await this.tryIceRestartRecovery(config)) {
|
|
277
|
+
this.connectionLossNotified = false;
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
await this.rebuildConnection(config);
|
|
281
|
+
this.connectionLossNotified = false;
|
|
282
|
+
} finally {
|
|
283
|
+
this.isRecovering = false;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
130
286
|
async sendText(text) {
|
|
287
|
+
if (this.isRecovering) {
|
|
288
|
+
throw new Error("Realtime transport reconnecting");
|
|
289
|
+
}
|
|
131
290
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
132
291
|
throw new Error("DataChannel not ready");
|
|
133
292
|
}
|
|
@@ -154,11 +313,17 @@ var OpenAIRealtimeClient = class {
|
|
|
154
313
|
this.beginResponseRequest();
|
|
155
314
|
}
|
|
156
315
|
async sendAudio(_audio) {
|
|
316
|
+
if (this.isRecovering) {
|
|
317
|
+
throw new Error("Realtime transport reconnecting");
|
|
318
|
+
}
|
|
157
319
|
console.warn(
|
|
158
320
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
159
321
|
);
|
|
160
322
|
}
|
|
161
323
|
async sendToolResult(callId, output) {
|
|
324
|
+
if (this.isRecovering) {
|
|
325
|
+
throw new Error("Realtime transport reconnecting");
|
|
326
|
+
}
|
|
162
327
|
const dc = this.requireOpenDataChannel();
|
|
163
328
|
dc.send(
|
|
164
329
|
JSON.stringify({
|
|
@@ -174,6 +339,9 @@ var OpenAIRealtimeClient = class {
|
|
|
174
339
|
this.beginResponseRequest();
|
|
175
340
|
}
|
|
176
341
|
async interrupt() {
|
|
342
|
+
if (this.isRecovering) {
|
|
343
|
+
throw new Error("Realtime transport reconnecting");
|
|
344
|
+
}
|
|
177
345
|
const dc = this.requireOpenDataChannel();
|
|
178
346
|
if (!this.isResponseInProgress) {
|
|
179
347
|
return;
|
|
@@ -191,7 +359,9 @@ var OpenAIRealtimeClient = class {
|
|
|
191
359
|
}
|
|
192
360
|
switch (event.type) {
|
|
193
361
|
case "session.created":
|
|
362
|
+
return;
|
|
194
363
|
case "session.updated":
|
|
364
|
+
this.resolvePendingSessionUpdate();
|
|
195
365
|
return;
|
|
196
366
|
case "response.audio.delta":
|
|
197
367
|
case "output_audio_buffer.started":
|
|
@@ -297,10 +467,16 @@ var OpenAIRealtimeClient = class {
|
|
|
297
467
|
case "error":
|
|
298
468
|
this.isResponseInProgress = false;
|
|
299
469
|
this.cancelInFlight = false;
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
470
|
+
{
|
|
471
|
+
const error = new Error(event.error?.message || "Unknown error");
|
|
472
|
+
if (this.isPendingSessionUpdateError(event)) {
|
|
473
|
+
this.rejectPendingSessionUpdate(error);
|
|
474
|
+
}
|
|
475
|
+
this.emitEvent({
|
|
476
|
+
type: "error",
|
|
477
|
+
error
|
|
478
|
+
});
|
|
479
|
+
}
|
|
304
480
|
return;
|
|
305
481
|
default:
|
|
306
482
|
return;
|
|
@@ -362,18 +538,133 @@ var OpenAIRealtimeClient = class {
|
|
|
362
538
|
}
|
|
363
539
|
return bootstrap;
|
|
364
540
|
}
|
|
541
|
+
bindConnectionEvents() {
|
|
542
|
+
if (!this.pc) {
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
546
|
+
const iceState = this.pc?.iceConnectionState;
|
|
547
|
+
if (iceState === "failed") {
|
|
548
|
+
this.iceDisconnectDebouncer.cancel();
|
|
549
|
+
this.emitConnectionLost("ice-failed");
|
|
550
|
+
} else if (iceState === "disconnected") {
|
|
551
|
+
this.iceDisconnectDebouncer.schedule();
|
|
552
|
+
} else {
|
|
553
|
+
this.iceDisconnectDebouncer.cancel();
|
|
554
|
+
}
|
|
555
|
+
});
|
|
556
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
557
|
+
const connectionState = this.pc?.connectionState;
|
|
558
|
+
if (connectionState === "failed" || connectionState === "closed") {
|
|
559
|
+
this.iceDisconnectDebouncer.cancel();
|
|
560
|
+
this.emitConnectionLost("connection-failed");
|
|
561
|
+
}
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
bindTransportLifecycleEvents() {
|
|
565
|
+
if (this.teardownTransportLifecycle) {
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
569
|
+
onHidden: this.handleHidden,
|
|
570
|
+
onOnline: this.handleOnline,
|
|
571
|
+
onPageHide: this.handlePageHide,
|
|
572
|
+
onPageShow: this.handlePageShow,
|
|
573
|
+
onVisible: this.handleVisible,
|
|
574
|
+
onDeviceChange: this.handleDeviceChange
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
unbindTransportLifecycleEvents() {
|
|
578
|
+
this.teardownTransportLifecycle?.();
|
|
579
|
+
this.teardownTransportLifecycle = void 0;
|
|
580
|
+
}
|
|
581
|
+
handleOnline = () => {
|
|
582
|
+
this.emitConnectionLost("online");
|
|
583
|
+
};
|
|
584
|
+
handleHidden = () => {
|
|
585
|
+
this.pauseAudioAnalysis();
|
|
586
|
+
};
|
|
587
|
+
handleVisible = () => {
|
|
588
|
+
this.resumeAudioAnalysis();
|
|
589
|
+
this.emitConnectionLost("visibility");
|
|
590
|
+
};
|
|
591
|
+
handlePageHide = () => {
|
|
592
|
+
this.pauseAudioAnalysis();
|
|
593
|
+
};
|
|
594
|
+
handlePageShow = (event) => {
|
|
595
|
+
this.resumeAudioAnalysis();
|
|
596
|
+
if (event.persisted) {
|
|
597
|
+
this.emitConnectionLost("pageshow");
|
|
598
|
+
}
|
|
599
|
+
};
|
|
600
|
+
handleDeviceChange = () => {
|
|
601
|
+
void this.refreshMicrophoneTrack();
|
|
602
|
+
};
|
|
603
|
+
async refreshMicrophoneTrack() {
|
|
604
|
+
if (!this.pc || this.isExplicitDisconnect) {
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
try {
|
|
608
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
609
|
+
audioSender: this.audioSender,
|
|
610
|
+
mediaStream: this.mediaStream,
|
|
611
|
+
peerConnection: this.pc
|
|
612
|
+
});
|
|
613
|
+
this.audioSender = audioSender;
|
|
614
|
+
this.mediaStream = mediaStream;
|
|
615
|
+
} catch (error) {
|
|
616
|
+
this.emitEvent({
|
|
617
|
+
type: "error",
|
|
618
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
emitConnectionLost(cause, error) {
|
|
623
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
this.connectionLossNotified = true;
|
|
627
|
+
this.isResponseInProgress = false;
|
|
628
|
+
this.cancelInFlight = false;
|
|
629
|
+
this.resetResponseTracking();
|
|
630
|
+
this.stopLipSyncAnalysis();
|
|
631
|
+
this.emitEvent({
|
|
632
|
+
type: "connection.lost",
|
|
633
|
+
cause,
|
|
634
|
+
error
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
needsRecovery(cause) {
|
|
638
|
+
if (!this.pc || !this.dc) {
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
642
|
+
return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
|
|
643
|
+
}
|
|
644
|
+
return true;
|
|
645
|
+
}
|
|
646
|
+
prepareAudioAnalysis() {
|
|
647
|
+
if (this.audioContext) {
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
651
|
+
if (!audioContextConstructor) {
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
this.audioContext = new audioContextConstructor();
|
|
655
|
+
}
|
|
365
656
|
setupAudioAnalysis(stream) {
|
|
366
657
|
try {
|
|
367
|
-
|
|
368
|
-
if (!
|
|
369
|
-
|
|
658
|
+
this.prepareAudioAnalysis();
|
|
659
|
+
if (!this.audioContext) {
|
|
660
|
+
return;
|
|
370
661
|
}
|
|
371
|
-
this.
|
|
372
|
-
|
|
662
|
+
this.audioSource?.disconnect();
|
|
663
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
373
664
|
this.analyser = this.audioContext.createAnalyser();
|
|
374
665
|
this.analyser.fftSize = 256;
|
|
375
666
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
376
|
-
|
|
667
|
+
this.audioSource.connect(this.analyser);
|
|
377
668
|
this.startLipSyncAnalysis();
|
|
378
669
|
} catch (error) {
|
|
379
670
|
console.error("Failed to setup audio analysis:", error);
|
|
@@ -409,8 +700,23 @@ var OpenAIRealtimeClient = class {
|
|
|
409
700
|
}
|
|
410
701
|
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
411
702
|
}
|
|
703
|
+
pauseAudioAnalysis() {
|
|
704
|
+
this.stopLipSyncAnalysis();
|
|
705
|
+
}
|
|
706
|
+
resumeAudioAnalysis() {
|
|
707
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
this.startLipSyncAnalysis();
|
|
711
|
+
}
|
|
412
712
|
cleanup() {
|
|
713
|
+
this.isCleaningUp = true;
|
|
714
|
+
this.unbindTransportLifecycleEvents();
|
|
715
|
+
this.iceDisconnectDebouncer.cancel();
|
|
413
716
|
this.stopLipSyncAnalysis();
|
|
717
|
+
this.rejectPendingSessionUpdate(
|
|
718
|
+
new Error("Realtime session ended before session update completed")
|
|
719
|
+
);
|
|
414
720
|
if (this.dc) {
|
|
415
721
|
this.dc.close();
|
|
416
722
|
this.dc = null;
|
|
@@ -431,9 +737,13 @@ var OpenAIRealtimeClient = class {
|
|
|
431
737
|
this.audioContext.close();
|
|
432
738
|
this.audioContext = null;
|
|
433
739
|
}
|
|
740
|
+
this.audioSource = null;
|
|
741
|
+
this.audioSender = null;
|
|
434
742
|
this.analyser = null;
|
|
435
743
|
this.isResponseInProgress = false;
|
|
436
744
|
this.cancelInFlight = false;
|
|
745
|
+
this.connectionLossNotified = false;
|
|
746
|
+
this.isCleaningUp = false;
|
|
437
747
|
this.resetResponseTracking();
|
|
438
748
|
}
|
|
439
749
|
beginResponseRequest() {
|
|
@@ -442,11 +752,86 @@ var OpenAIRealtimeClient = class {
|
|
|
442
752
|
this.cancelInFlight = false;
|
|
443
753
|
}
|
|
444
754
|
requireOpenDataChannel() {
|
|
755
|
+
if (this.isRecovering) {
|
|
756
|
+
throw new Error("Realtime transport reconnecting");
|
|
757
|
+
}
|
|
445
758
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
446
759
|
throw new Error("DataChannel not ready");
|
|
447
760
|
}
|
|
448
761
|
return this.dc;
|
|
449
762
|
}
|
|
763
|
+
async tryIceRestartRecovery(config) {
|
|
764
|
+
const pc = this.pc;
|
|
765
|
+
if (!pc || this.shouldRebuildConnection()) {
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
if (pc.iceConnectionState === "connected") {
|
|
769
|
+
return true;
|
|
770
|
+
}
|
|
771
|
+
if (pc.iceConnectionState === "disconnected") {
|
|
772
|
+
await delay(1500);
|
|
773
|
+
if (pc.iceConnectionState !== "disconnected") {
|
|
774
|
+
return true;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
778
|
+
return false;
|
|
779
|
+
}
|
|
780
|
+
if (typeof pc.restartIce === "function") {
|
|
781
|
+
pc.restartIce();
|
|
782
|
+
}
|
|
783
|
+
const offer = await pc.createOffer({ iceRestart: true });
|
|
784
|
+
await pc.setLocalDescription(offer);
|
|
785
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
786
|
+
transport: "webrtc",
|
|
787
|
+
session: config ?? this.currentSessionConfig ?? {},
|
|
788
|
+
sdpOffer: offer.sdp
|
|
789
|
+
});
|
|
790
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
791
|
+
throw new Error(
|
|
792
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
793
|
+
);
|
|
794
|
+
}
|
|
795
|
+
await pc.setRemoteDescription({
|
|
796
|
+
type: "answer",
|
|
797
|
+
sdp: bootstrap.answerSdp
|
|
798
|
+
});
|
|
799
|
+
return true;
|
|
800
|
+
}
|
|
801
|
+
shouldRebuildConnection() {
|
|
802
|
+
return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
|
|
803
|
+
}
|
|
804
|
+
async rebuildConnection(config) {
|
|
805
|
+
this.connectionLossNotified = true;
|
|
806
|
+
this.cleanup();
|
|
807
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
808
|
+
}
|
|
809
|
+
resolvePendingSessionUpdate() {
|
|
810
|
+
if (!this.pendingSessionUpdate) {
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
814
|
+
this.pendingSessionUpdate.resolve();
|
|
815
|
+
this.pendingSessionUpdate = null;
|
|
816
|
+
}
|
|
817
|
+
rejectPendingSessionUpdate(error) {
|
|
818
|
+
if (!this.pendingSessionUpdate) {
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
822
|
+
this.pendingSessionUpdate.reject(error);
|
|
823
|
+
this.pendingSessionUpdate = null;
|
|
824
|
+
}
|
|
825
|
+
isPendingSessionUpdateError(event) {
|
|
826
|
+
const pendingUpdate = this.pendingSessionUpdate;
|
|
827
|
+
if (!pendingUpdate) {
|
|
828
|
+
return false;
|
|
829
|
+
}
|
|
830
|
+
return event.error?.event_id === pendingUpdate.eventId;
|
|
831
|
+
}
|
|
832
|
+
getSessionUpdateTimeoutMs() {
|
|
833
|
+
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
834
|
+
}
|
|
450
835
|
resetResponseTracking() {
|
|
451
836
|
this.assistantText = "";
|
|
452
837
|
this.hasStartedAssistantResponse = false;
|
|
@@ -472,6 +857,34 @@ var OpenAIRealtimeClient = class {
|
|
|
472
857
|
function createOpenAIRealtimeClient(options) {
|
|
473
858
|
return new OpenAIRealtimeClient(options);
|
|
474
859
|
}
|
|
860
|
+
function toOpenAIRealtimeSessionUpdate(config) {
|
|
861
|
+
const session = {
|
|
862
|
+
audio: {
|
|
863
|
+
output: {
|
|
864
|
+
voice: config?.voice ?? DEFAULT_REALTIME_VOICE
|
|
865
|
+
}
|
|
866
|
+
},
|
|
867
|
+
tool_choice: config?.toolChoice ?? "auto"
|
|
868
|
+
};
|
|
869
|
+
if (config?.instructions) {
|
|
870
|
+
session.instructions = config.instructions;
|
|
871
|
+
}
|
|
872
|
+
if (config?.temperature !== void 0) {
|
|
873
|
+
session.temperature = config.temperature;
|
|
874
|
+
}
|
|
875
|
+
if (config?.maxTokens !== void 0) {
|
|
876
|
+
session.max_response_output_tokens = config.maxTokens;
|
|
877
|
+
}
|
|
878
|
+
if (config?.tools?.length) {
|
|
879
|
+
session.tools = config.tools;
|
|
880
|
+
}
|
|
881
|
+
return session;
|
|
882
|
+
}
|
|
883
|
+
function delay(ms) {
|
|
884
|
+
return new Promise((resolve) => {
|
|
885
|
+
setTimeout(resolve, ms);
|
|
886
|
+
});
|
|
887
|
+
}
|
|
475
888
|
|
|
476
889
|
exports.OpenAIRealtimeClient = OpenAIRealtimeClient;
|
|
477
890
|
exports.createOpenAIRealtimeClient = createOpenAIRealtimeClient;
|