@charivo/realtime 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -7
- package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
- package/dist/chunk-DHNRDHST.mjs.map +1 -0
- package/dist/chunk-PDODY2T7.mjs +65 -0
- package/dist/chunk-PDODY2T7.mjs.map +1 -0
- package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
- package/dist/chunk-PTFZ7CML.mjs.map +1 -0
- package/dist/chunk-Y7JO2FFX.mjs +121 -0
- package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
- package/dist/index.d.mts +16 -4
- package/dist/index.d.ts +16 -4
- package/dist/index.js +240 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +238 -98
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +37 -1
- package/dist/openai/index.d.ts +37 -1
- package/dist/openai/index.js +427 -14
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.d.mts +26 -1
- package/dist/openai-agents/index.d.ts +26 -1
- package/dist/openai-agents/index.js +311 -18
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -2
- package/dist/remote/index.d.mts +3 -1
- package/dist/remote/index.d.ts +3 -1
- package/dist/remote/index.js +663 -32
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +10 -3
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
- package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
- package/package.json +3 -3
- package/dist/chunk-IIKECRXN.mjs.map +0 -1
- package/dist/chunk-JGJZKKMH.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/remote/index.js
CHANGED
|
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
|
|
|
5
5
|
|
|
6
6
|
// src/remote/index.ts
|
|
7
7
|
|
|
8
|
+
// src/internal/microphone.ts
|
|
9
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
10
|
+
echoCancellation: true,
|
|
11
|
+
noiseSuppression: true,
|
|
12
|
+
autoGainControl: true
|
|
13
|
+
};
|
|
14
|
+
async function acquireMicrophoneStream() {
|
|
15
|
+
return navigator.mediaDevices.getUserMedia({
|
|
16
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
20
|
+
function bindTransportLifecycle(callbacks) {
|
|
21
|
+
const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
22
|
+
onHidden: callbacks.onHidden,
|
|
23
|
+
onOnline: callbacks.onOnline,
|
|
24
|
+
onPageHide: callbacks.onPageHide,
|
|
25
|
+
onPageShow: callbacks.onPageShow,
|
|
26
|
+
onVisible: callbacks.onVisible
|
|
27
|
+
});
|
|
28
|
+
if (callbacks.onDeviceChange) {
|
|
29
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
30
|
+
"devicechange",
|
|
31
|
+
callbacks.onDeviceChange
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return () => {
|
|
35
|
+
teardownBrowserLifecycle();
|
|
36
|
+
if (callbacks.onDeviceChange) {
|
|
37
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
38
|
+
"devicechange",
|
|
39
|
+
callbacks.onDeviceChange
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
async function replaceMicrophoneTrack(state) {
|
|
45
|
+
if (!state.peerConnection) {
|
|
46
|
+
throw new Error("Realtime transport not active");
|
|
47
|
+
}
|
|
48
|
+
const nextStream = await acquireMicrophoneStream();
|
|
49
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
50
|
+
if (!nextTrack) {
|
|
51
|
+
throw new Error("Microphone access required for Realtime API");
|
|
52
|
+
}
|
|
53
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
54
|
+
if (!sender) {
|
|
55
|
+
throw new Error("No outbound audio sender available");
|
|
56
|
+
}
|
|
57
|
+
await sender.replaceTrack(nextTrack);
|
|
58
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
59
|
+
return {
|
|
60
|
+
audioSender: sender,
|
|
61
|
+
mediaStream: nextStream
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
65
|
+
let pending = null;
|
|
66
|
+
return {
|
|
67
|
+
schedule() {
|
|
68
|
+
if (pending) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
pending = setTimeout(() => {
|
|
72
|
+
pending = null;
|
|
73
|
+
onDisconnect();
|
|
74
|
+
}, delayMs);
|
|
75
|
+
},
|
|
76
|
+
cancel() {
|
|
77
|
+
if (!pending) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
clearTimeout(pending);
|
|
81
|
+
pending = null;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
8
86
|
// src/internal/shared.ts
|
|
9
87
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
10
88
|
function isRecord(value) {
|
|
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
|
|
|
83
161
|
}
|
|
84
162
|
audioAnalysisStream = null;
|
|
85
163
|
audioContext = null;
|
|
164
|
+
audioSource = null;
|
|
86
165
|
analyser = null;
|
|
87
166
|
lipSyncInterval = null;
|
|
88
167
|
pendingAudioElementPoll = null;
|
|
89
168
|
observedAudioElement = null;
|
|
90
169
|
observedAudioElementListener = null;
|
|
170
|
+
async prepareAudioContext() {
|
|
171
|
+
if (this.audioContext) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
175
|
+
if (!audioContextConstructor) {
|
|
176
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
177
|
+
}
|
|
178
|
+
this.audioContext = new audioContextConstructor();
|
|
179
|
+
}
|
|
91
180
|
observeAudioElement(audioElement) {
|
|
92
181
|
this.detachObservedAudioElement();
|
|
93
182
|
const tryAttachStream = () => {
|
|
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
|
|
|
109
198
|
this.stopLipSyncAnalysis();
|
|
110
199
|
this.audioAnalysisStream = stream;
|
|
111
200
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
201
|
+
if (!this.audioContext) {
|
|
202
|
+
void this.prepareAudioContext().catch((error) => {
|
|
203
|
+
this.options.onError?.(error);
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (!this.audioContext) {
|
|
207
|
+
return;
|
|
115
208
|
}
|
|
116
|
-
this.
|
|
117
|
-
|
|
209
|
+
this.audioSource?.disconnect?.();
|
|
210
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
118
211
|
this.analyser = this.audioContext.createAnalyser();
|
|
119
212
|
this.analyser.fftSize = 256;
|
|
120
213
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
121
|
-
|
|
214
|
+
this.audioSource.connect(this.analyser);
|
|
122
215
|
this.startLipSyncAnalysis();
|
|
123
216
|
} catch (error) {
|
|
124
217
|
this.options.onError?.(error);
|
|
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
|
|
|
127
220
|
stopOutput() {
|
|
128
221
|
this.stopLipSyncAnalysis();
|
|
129
222
|
}
|
|
223
|
+
pause() {
|
|
224
|
+
this.stopLipSyncAnalysis();
|
|
225
|
+
}
|
|
226
|
+
resume() {
|
|
227
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
this.startLipSyncAnalysis();
|
|
231
|
+
}
|
|
130
232
|
cleanup() {
|
|
131
233
|
this.stopLipSyncAnalysis();
|
|
132
234
|
this.stopAudioElementPolling();
|
|
133
235
|
this.detachObservedAudioElement();
|
|
236
|
+
this.audioSource?.disconnect?.();
|
|
237
|
+
this.audioSource = null;
|
|
134
238
|
if (this.audioContext) {
|
|
135
239
|
void this.audioContext.close();
|
|
136
240
|
this.audioContext = null;
|
|
@@ -183,18 +287,20 @@ var LipSyncAnalyzer = class {
|
|
|
183
287
|
}
|
|
184
288
|
};
|
|
185
289
|
|
|
290
|
+
// src/instructions.ts
|
|
291
|
+
var DEFAULT_REALTIME_VOICE = "marin";
|
|
292
|
+
|
|
186
293
|
// src/openai-agents/session-config.ts
|
|
187
294
|
var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
|
|
188
295
|
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
189
|
-
var DEFAULT_VOICE = "marin";
|
|
190
296
|
function resolveInstructions(config) {
|
|
191
297
|
return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
|
|
192
298
|
}
|
|
193
299
|
function resolveVoice(config) {
|
|
194
|
-
return config?.voice ??
|
|
300
|
+
return config?.voice ?? DEFAULT_REALTIME_VOICE;
|
|
195
301
|
}
|
|
196
302
|
function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
197
|
-
|
|
303
|
+
const sessionConfig = {
|
|
198
304
|
model: config?.model ?? DEFAULT_MODEL,
|
|
199
305
|
instructions: resolveInstructions(config),
|
|
200
306
|
toolChoice: config?.toolChoice ?? "auto",
|
|
@@ -205,6 +311,13 @@ function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
|
205
311
|
}
|
|
206
312
|
}
|
|
207
313
|
};
|
|
314
|
+
if (config?.temperature !== void 0) {
|
|
315
|
+
sessionConfig.temperature = config.temperature;
|
|
316
|
+
}
|
|
317
|
+
if (config?.maxTokens !== void 0) {
|
|
318
|
+
sessionConfig.maxResponseOutputTokens = config.maxTokens;
|
|
319
|
+
}
|
|
320
|
+
return sessionConfig;
|
|
208
321
|
}
|
|
209
322
|
|
|
210
323
|
// src/openai-agents/tool-schema.ts
|
|
@@ -256,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
256
369
|
session = null;
|
|
257
370
|
transport = null;
|
|
258
371
|
audioElement = null;
|
|
372
|
+
mediaStream = null;
|
|
373
|
+
peerConnection = null;
|
|
374
|
+
audioSender = null;
|
|
259
375
|
connectionWasActive = false;
|
|
376
|
+
connectionLossNotified = false;
|
|
377
|
+
isExplicitDisconnect = false;
|
|
378
|
+
isRecovering = false;
|
|
379
|
+
isCleaningUp = false;
|
|
260
380
|
assistant = { text: "", started: false };
|
|
261
381
|
latestHistory = [];
|
|
382
|
+
currentSessionConfig;
|
|
383
|
+
teardownTransportLifecycle;
|
|
384
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
385
|
+
this.emitConnectionLost("ice-disconnected");
|
|
386
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
262
387
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
263
388
|
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
264
389
|
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
@@ -271,18 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
271
396
|
});
|
|
272
397
|
async connect(config) {
|
|
273
398
|
try {
|
|
399
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
400
|
+
throw new Error("Realtime session disconnect requested");
|
|
401
|
+
}
|
|
274
402
|
this.log("Starting OpenAI Agents Realtime WebRTC connection");
|
|
403
|
+
this.isExplicitDisconnect = false;
|
|
404
|
+
this.isRecovering = false;
|
|
405
|
+
this.connectionLossNotified = false;
|
|
406
|
+
this.currentSessionConfig = config;
|
|
275
407
|
this.audioElement = document.createElement("audio");
|
|
276
408
|
this.audioElement.autoplay = true;
|
|
277
409
|
this.audioElement.setAttribute("playsinline", "true");
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
voice: resolveVoice(config)
|
|
283
|
-
});
|
|
410
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
411
|
+
this.bindTransportLifecycleEvents();
|
|
412
|
+
await this.lipSyncAnalyzer.prepareAudioContext();
|
|
413
|
+
const agent = this.createAgent(config);
|
|
284
414
|
this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
|
|
285
|
-
this.createTransportOptions(this.audioElement)
|
|
415
|
+
this.createTransportOptions(this.audioElement, this.mediaStream)
|
|
286
416
|
);
|
|
287
417
|
this.session = new agentsRealtime.RealtimeSession(agent, {
|
|
288
418
|
transport: this.transport,
|
|
@@ -314,9 +444,37 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
314
444
|
}
|
|
315
445
|
async disconnect() {
|
|
316
446
|
this.log("Disconnecting OpenAI Agents Realtime WebRTC");
|
|
447
|
+
this.isExplicitDisconnect = true;
|
|
448
|
+
this.isRecovering = false;
|
|
317
449
|
this.cleanup();
|
|
318
450
|
}
|
|
451
|
+
async updateSession(config) {
|
|
452
|
+
if (!this.session) {
|
|
453
|
+
throw new Error("Realtime session not active");
|
|
454
|
+
}
|
|
455
|
+
this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
|
|
456
|
+
await this.session.updateAgent(this.createAgent(config));
|
|
457
|
+
this.currentSessionConfig = config;
|
|
458
|
+
}
|
|
459
|
+
async recover(config) {
|
|
460
|
+
this.isRecovering = true;
|
|
461
|
+
this.connectionLossNotified = true;
|
|
462
|
+
this.currentSessionConfig = config;
|
|
463
|
+
try {
|
|
464
|
+
this.cleanupPendingToolCalls(
|
|
465
|
+
new Error("Realtime session interrupted during reconnect")
|
|
466
|
+
);
|
|
467
|
+
this.cleanup();
|
|
468
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
469
|
+
this.connectionLossNotified = false;
|
|
470
|
+
} finally {
|
|
471
|
+
this.isRecovering = false;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
319
474
|
async sendText(text) {
|
|
475
|
+
if (this.isRecovering) {
|
|
476
|
+
throw new Error("Realtime transport reconnecting");
|
|
477
|
+
}
|
|
320
478
|
if (!this.session) {
|
|
321
479
|
throw new Error("Realtime session not active");
|
|
322
480
|
}
|
|
@@ -324,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
324
482
|
this.session.sendMessage(text);
|
|
325
483
|
}
|
|
326
484
|
async sendAudio(_audio) {
|
|
485
|
+
if (this.isRecovering) {
|
|
486
|
+
throw new Error("Realtime transport reconnecting");
|
|
487
|
+
}
|
|
327
488
|
console.warn(
|
|
328
489
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
329
490
|
);
|
|
330
491
|
}
|
|
331
492
|
async sendToolResult(callId, output) {
|
|
493
|
+
if (this.isRecovering) {
|
|
494
|
+
throw new Error("Realtime transport reconnecting");
|
|
495
|
+
}
|
|
332
496
|
const pendingCall = this.pendingToolCalls.get(callId);
|
|
333
497
|
if (!pendingCall) {
|
|
334
498
|
throw new Error(`No pending realtime tool call for "${callId}"`);
|
|
@@ -337,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
337
501
|
pendingCall.resolve(output);
|
|
338
502
|
}
|
|
339
503
|
async interrupt() {
|
|
504
|
+
if (this.isRecovering) {
|
|
505
|
+
throw new Error("Realtime transport reconnecting");
|
|
506
|
+
}
|
|
340
507
|
if (!this.session) {
|
|
341
508
|
throw new Error("Realtime session not active");
|
|
342
509
|
}
|
|
@@ -346,6 +513,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
346
513
|
onEvent(callback) {
|
|
347
514
|
this.eventCallbacks.add(callback);
|
|
348
515
|
}
|
|
516
|
+
createAgent(config) {
|
|
517
|
+
return new agentsRealtime.RealtimeAgent({
|
|
518
|
+
name: "charivo-realtime-agent",
|
|
519
|
+
instructions: resolveInstructions(config),
|
|
520
|
+
tools: this.createProxyTools(config?.tools),
|
|
521
|
+
voice: resolveVoice(config)
|
|
522
|
+
});
|
|
523
|
+
}
|
|
349
524
|
bindSessionEvents(session) {
|
|
350
525
|
session.on("agent_start", () => {
|
|
351
526
|
this.ensureAssistantStarted();
|
|
@@ -392,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
392
567
|
if (status === "disconnected" && this.connectionWasActive) {
|
|
393
568
|
this.cleanupPendingToolCalls();
|
|
394
569
|
this.connectionWasActive = false;
|
|
395
|
-
this.
|
|
570
|
+
this.emitConnectionLost("connection-failed");
|
|
396
571
|
}
|
|
397
572
|
});
|
|
398
573
|
}
|
|
@@ -494,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
494
669
|
});
|
|
495
670
|
});
|
|
496
671
|
}
|
|
497
|
-
createTransportOptions(audioElement) {
|
|
672
|
+
createTransportOptions(audioElement, mediaStream) {
|
|
498
673
|
return {
|
|
499
674
|
audioElement,
|
|
675
|
+
mediaStream,
|
|
500
676
|
changePeerConnection: async (peerConnection) => {
|
|
677
|
+
this.peerConnection = peerConnection;
|
|
678
|
+
this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
|
|
679
|
+
this.bindPeerConnectionEvents(peerConnection);
|
|
501
680
|
peerConnection.addEventListener("track", (event) => {
|
|
502
681
|
const stream = event.streams[0];
|
|
503
682
|
if (stream) {
|
|
@@ -508,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
508
687
|
}
|
|
509
688
|
};
|
|
510
689
|
}
|
|
690
|
+
bindPeerConnectionEvents(peerConnection) {
|
|
691
|
+
peerConnection.addEventListener("iceconnectionstatechange", () => {
|
|
692
|
+
const iceState = peerConnection.iceConnectionState;
|
|
693
|
+
if (iceState === "failed") {
|
|
694
|
+
this.iceDisconnectDebouncer.cancel();
|
|
695
|
+
this.emitConnectionLost("ice-failed");
|
|
696
|
+
} else if (iceState === "disconnected") {
|
|
697
|
+
this.iceDisconnectDebouncer.schedule();
|
|
698
|
+
} else {
|
|
699
|
+
this.iceDisconnectDebouncer.cancel();
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
peerConnection.addEventListener("connectionstatechange", () => {
|
|
703
|
+
if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
|
|
704
|
+
this.iceDisconnectDebouncer.cancel();
|
|
705
|
+
this.emitConnectionLost("connection-failed");
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
bindTransportLifecycleEvents() {
|
|
710
|
+
if (this.teardownTransportLifecycle) {
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
714
|
+
onHidden: this.handleHidden,
|
|
715
|
+
onOnline: this.handleOnline,
|
|
716
|
+
onPageHide: this.handlePageHide,
|
|
717
|
+
onPageShow: this.handlePageShow,
|
|
718
|
+
onVisible: this.handleVisible,
|
|
719
|
+
onDeviceChange: this.handleDeviceChange
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
unbindTransportLifecycleEvents() {
|
|
723
|
+
this.teardownTransportLifecycle?.();
|
|
724
|
+
this.teardownTransportLifecycle = void 0;
|
|
725
|
+
}
|
|
726
|
+
handleOnline = () => {
|
|
727
|
+
this.emitConnectionLost("online");
|
|
728
|
+
};
|
|
729
|
+
handleHidden = () => {
|
|
730
|
+
this.lipSyncAnalyzer.pause();
|
|
731
|
+
};
|
|
732
|
+
handleVisible = () => {
|
|
733
|
+
this.lipSyncAnalyzer.resume();
|
|
734
|
+
this.emitConnectionLost("visibility");
|
|
735
|
+
};
|
|
736
|
+
handlePageHide = () => {
|
|
737
|
+
this.lipSyncAnalyzer.pause();
|
|
738
|
+
};
|
|
739
|
+
handlePageShow = (event) => {
|
|
740
|
+
this.lipSyncAnalyzer.resume();
|
|
741
|
+
if (event.persisted) {
|
|
742
|
+
this.emitConnectionLost("pageshow");
|
|
743
|
+
}
|
|
744
|
+
};
|
|
745
|
+
handleDeviceChange = () => {
|
|
746
|
+
void this.refreshMicrophoneTrack();
|
|
747
|
+
};
|
|
748
|
+
async refreshMicrophoneTrack() {
|
|
749
|
+
if (!this.peerConnection || this.isExplicitDisconnect) {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
try {
|
|
753
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
754
|
+
audioSender: this.audioSender,
|
|
755
|
+
mediaStream: this.mediaStream,
|
|
756
|
+
peerConnection: this.peerConnection
|
|
757
|
+
});
|
|
758
|
+
this.audioSender = audioSender;
|
|
759
|
+
this.mediaStream = mediaStream;
|
|
760
|
+
} catch (error) {
|
|
761
|
+
this.emitEvent({
|
|
762
|
+
type: "error",
|
|
763
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
emitConnectionLost(cause, error) {
|
|
768
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
this.connectionLossNotified = true;
|
|
772
|
+
this.cleanupPendingToolCalls(
|
|
773
|
+
new Error("Realtime session interrupted during reconnect")
|
|
774
|
+
);
|
|
775
|
+
this.resetAssistantTracking();
|
|
776
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
777
|
+
this.emitEvent({
|
|
778
|
+
type: "connection.lost",
|
|
779
|
+
cause,
|
|
780
|
+
error
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
shouldRecover(cause) {
|
|
784
|
+
if (!this.peerConnection) {
|
|
785
|
+
return false;
|
|
786
|
+
}
|
|
787
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
788
|
+
return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
|
|
789
|
+
}
|
|
790
|
+
return true;
|
|
791
|
+
}
|
|
511
792
|
cleanup(error) {
|
|
793
|
+
this.connectionWasActive = false;
|
|
794
|
+
this.isCleaningUp = true;
|
|
795
|
+
this.iceDisconnectDebouncer.cancel();
|
|
796
|
+
this.unbindTransportLifecycleEvents();
|
|
512
797
|
this.cleanupPendingToolCalls(error);
|
|
513
798
|
this.lipSyncAnalyzer.cleanup();
|
|
514
799
|
if (this.transport) {
|
|
@@ -523,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
523
808
|
this.audioElement.srcObject = null;
|
|
524
809
|
this.audioElement = null;
|
|
525
810
|
}
|
|
811
|
+
if (this.mediaStream) {
|
|
812
|
+
this.mediaStream.getTracks?.().forEach((track) => track.stop());
|
|
813
|
+
this.mediaStream = null;
|
|
814
|
+
}
|
|
815
|
+
this.peerConnection = null;
|
|
816
|
+
this.audioSender = null;
|
|
526
817
|
this.latestHistory = [];
|
|
818
|
+
this.connectionLossNotified = false;
|
|
819
|
+
this.isCleaningUp = false;
|
|
527
820
|
this.resetAssistantTracking();
|
|
528
821
|
}
|
|
529
822
|
resetAssistantTracking() {
|
|
@@ -569,6 +862,7 @@ var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
|
|
|
569
862
|
"conversation.item.input_audio_transcription.completed",
|
|
570
863
|
"error"
|
|
571
864
|
]);
|
|
865
|
+
var DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5e3;
|
|
572
866
|
var OpenAIRealtimeClient = class {
|
|
573
867
|
constructor(options = {}) {
|
|
574
868
|
this.options = options;
|
|
@@ -577,7 +871,9 @@ var OpenAIRealtimeClient = class {
|
|
|
577
871
|
dc = null;
|
|
578
872
|
audioElement = null;
|
|
579
873
|
mediaStream = null;
|
|
874
|
+
audioSender = null;
|
|
580
875
|
audioContext = null;
|
|
876
|
+
audioSource = null;
|
|
581
877
|
analyser = null;
|
|
582
878
|
lipSyncInterval = null;
|
|
583
879
|
isResponseInProgress = false;
|
|
@@ -585,13 +881,35 @@ var OpenAIRealtimeClient = class {
|
|
|
585
881
|
hasStartedAssistantResponse = false;
|
|
586
882
|
hasStartedAudioOutput = false;
|
|
587
883
|
assistantText = "";
|
|
884
|
+
nextSessionUpdateId = 1;
|
|
885
|
+
pendingSessionUpdate = null;
|
|
886
|
+
currentSessionConfig;
|
|
887
|
+
connectionLossNotified = false;
|
|
888
|
+
isExplicitDisconnect = false;
|
|
889
|
+
isRecovering = false;
|
|
890
|
+
isCleaningUp = false;
|
|
891
|
+
teardownTransportLifecycle;
|
|
892
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
893
|
+
this.emitConnectionLost("ice-disconnected");
|
|
894
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
588
895
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
589
896
|
async connect(config) {
|
|
590
897
|
try {
|
|
898
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
899
|
+
throw new Error("Realtime session disconnect requested");
|
|
900
|
+
}
|
|
591
901
|
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
902
|
+
this.isExplicitDisconnect = false;
|
|
903
|
+
this.isRecovering = false;
|
|
904
|
+
this.connectionLossNotified = false;
|
|
905
|
+
this.currentSessionConfig = config;
|
|
592
906
|
this.pc = new RTCPeerConnection();
|
|
593
907
|
this.audioElement = document.createElement("audio");
|
|
594
908
|
this.audioElement.autoplay = true;
|
|
909
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
910
|
+
this.prepareAudioAnalysis();
|
|
911
|
+
this.bindConnectionEvents();
|
|
912
|
+
this.bindTransportLifecycleEvents();
|
|
595
913
|
this.pc.ontrack = (event) => {
|
|
596
914
|
if (this.audioElement) {
|
|
597
915
|
this.audioElement.srcObject = event.streams[0];
|
|
@@ -599,11 +917,9 @@ var OpenAIRealtimeClient = class {
|
|
|
599
917
|
this.setupAudioAnalysis(event.streams[0]);
|
|
600
918
|
};
|
|
601
919
|
try {
|
|
602
|
-
this.mediaStream = await
|
|
603
|
-
audio: true
|
|
604
|
-
});
|
|
920
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
605
921
|
const audioTrack = this.mediaStream.getTracks()[0];
|
|
606
|
-
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
922
|
+
this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
|
|
607
923
|
} catch {
|
|
608
924
|
throw new Error("Microphone access required for Realtime API");
|
|
609
925
|
}
|
|
@@ -642,9 +958,64 @@ var OpenAIRealtimeClient = class {
|
|
|
642
958
|
}
|
|
643
959
|
async disconnect() {
|
|
644
960
|
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
961
|
+
this.isExplicitDisconnect = true;
|
|
962
|
+
this.isRecovering = false;
|
|
645
963
|
this.cleanup();
|
|
646
964
|
}
|
|
965
|
+
async updateSession(config) {
|
|
966
|
+
if (this.isResponseInProgress) {
|
|
967
|
+
throw new Error(
|
|
968
|
+
"Cannot update the realtime session while a response is in progress. Call interrupt() first."
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
const dc = this.requireOpenDataChannel();
|
|
972
|
+
const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;
|
|
973
|
+
if (this.pendingSessionUpdate) {
|
|
974
|
+
throw new Error("Realtime session update already in progress");
|
|
975
|
+
}
|
|
976
|
+
dc.send(
|
|
977
|
+
JSON.stringify({
|
|
978
|
+
type: "session.update",
|
|
979
|
+
event_id: eventId,
|
|
980
|
+
session: toOpenAIRealtimeSessionUpdate(config)
|
|
981
|
+
})
|
|
982
|
+
);
|
|
983
|
+
await new Promise((resolve, reject) => {
|
|
984
|
+
const timeoutId = setTimeout(() => {
|
|
985
|
+
this.rejectPendingSessionUpdate(
|
|
986
|
+
new Error(
|
|
987
|
+
`Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`
|
|
988
|
+
)
|
|
989
|
+
);
|
|
990
|
+
}, this.getSessionUpdateTimeoutMs());
|
|
991
|
+
this.pendingSessionUpdate = {
|
|
992
|
+
eventId,
|
|
993
|
+
resolve,
|
|
994
|
+
reject,
|
|
995
|
+
timeoutId
|
|
996
|
+
};
|
|
997
|
+
});
|
|
998
|
+
this.currentSessionConfig = config;
|
|
999
|
+
}
|
|
1000
|
+
async recover(config) {
|
|
1001
|
+
this.isRecovering = true;
|
|
1002
|
+
this.connectionLossNotified = true;
|
|
1003
|
+
this.currentSessionConfig = config;
|
|
1004
|
+
try {
|
|
1005
|
+
if (await this.tryIceRestartRecovery(config)) {
|
|
1006
|
+
this.connectionLossNotified = false;
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
await this.rebuildConnection(config);
|
|
1010
|
+
this.connectionLossNotified = false;
|
|
1011
|
+
} finally {
|
|
1012
|
+
this.isRecovering = false;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
647
1015
|
async sendText(text) {
|
|
1016
|
+
if (this.isRecovering) {
|
|
1017
|
+
throw new Error("Realtime transport reconnecting");
|
|
1018
|
+
}
|
|
648
1019
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
649
1020
|
throw new Error("DataChannel not ready");
|
|
650
1021
|
}
|
|
@@ -671,11 +1042,17 @@ var OpenAIRealtimeClient = class {
|
|
|
671
1042
|
this.beginResponseRequest();
|
|
672
1043
|
}
|
|
673
1044
|
async sendAudio(_audio) {
|
|
1045
|
+
if (this.isRecovering) {
|
|
1046
|
+
throw new Error("Realtime transport reconnecting");
|
|
1047
|
+
}
|
|
674
1048
|
console.warn(
|
|
675
1049
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
676
1050
|
);
|
|
677
1051
|
}
|
|
678
1052
|
async sendToolResult(callId, output) {
|
|
1053
|
+
if (this.isRecovering) {
|
|
1054
|
+
throw new Error("Realtime transport reconnecting");
|
|
1055
|
+
}
|
|
679
1056
|
const dc = this.requireOpenDataChannel();
|
|
680
1057
|
dc.send(
|
|
681
1058
|
JSON.stringify({
|
|
@@ -691,6 +1068,9 @@ var OpenAIRealtimeClient = class {
|
|
|
691
1068
|
this.beginResponseRequest();
|
|
692
1069
|
}
|
|
693
1070
|
async interrupt() {
|
|
1071
|
+
if (this.isRecovering) {
|
|
1072
|
+
throw new Error("Realtime transport reconnecting");
|
|
1073
|
+
}
|
|
694
1074
|
const dc = this.requireOpenDataChannel();
|
|
695
1075
|
if (!this.isResponseInProgress) {
|
|
696
1076
|
return;
|
|
@@ -708,7 +1088,9 @@ var OpenAIRealtimeClient = class {
|
|
|
708
1088
|
}
|
|
709
1089
|
switch (event.type) {
|
|
710
1090
|
case "session.created":
|
|
1091
|
+
return;
|
|
711
1092
|
case "session.updated":
|
|
1093
|
+
this.resolvePendingSessionUpdate();
|
|
712
1094
|
return;
|
|
713
1095
|
case "response.audio.delta":
|
|
714
1096
|
case "output_audio_buffer.started":
|
|
@@ -814,10 +1196,16 @@ var OpenAIRealtimeClient = class {
|
|
|
814
1196
|
case "error":
|
|
815
1197
|
this.isResponseInProgress = false;
|
|
816
1198
|
this.cancelInFlight = false;
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
1199
|
+
{
|
|
1200
|
+
const error = new Error(event.error?.message || "Unknown error");
|
|
1201
|
+
if (this.isPendingSessionUpdateError(event)) {
|
|
1202
|
+
this.rejectPendingSessionUpdate(error);
|
|
1203
|
+
}
|
|
1204
|
+
this.emitEvent({
|
|
1205
|
+
type: "error",
|
|
1206
|
+
error
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
821
1209
|
return;
|
|
822
1210
|
default:
|
|
823
1211
|
return;
|
|
@@ -879,18 +1267,133 @@ var OpenAIRealtimeClient = class {
|
|
|
879
1267
|
}
|
|
880
1268
|
return bootstrap;
|
|
881
1269
|
}
|
|
1270
|
+
bindConnectionEvents() {
|
|
1271
|
+
if (!this.pc) {
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
1275
|
+
const iceState = this.pc?.iceConnectionState;
|
|
1276
|
+
if (iceState === "failed") {
|
|
1277
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1278
|
+
this.emitConnectionLost("ice-failed");
|
|
1279
|
+
} else if (iceState === "disconnected") {
|
|
1280
|
+
this.iceDisconnectDebouncer.schedule();
|
|
1281
|
+
} else {
|
|
1282
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1283
|
+
}
|
|
1284
|
+
});
|
|
1285
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
1286
|
+
const connectionState = this.pc?.connectionState;
|
|
1287
|
+
if (connectionState === "failed" || connectionState === "closed") {
|
|
1288
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1289
|
+
this.emitConnectionLost("connection-failed");
|
|
1290
|
+
}
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
bindTransportLifecycleEvents() {
|
|
1294
|
+
if (this.teardownTransportLifecycle) {
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
1298
|
+
onHidden: this.handleHidden,
|
|
1299
|
+
onOnline: this.handleOnline,
|
|
1300
|
+
onPageHide: this.handlePageHide,
|
|
1301
|
+
onPageShow: this.handlePageShow,
|
|
1302
|
+
onVisible: this.handleVisible,
|
|
1303
|
+
onDeviceChange: this.handleDeviceChange
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
unbindTransportLifecycleEvents() {
|
|
1307
|
+
this.teardownTransportLifecycle?.();
|
|
1308
|
+
this.teardownTransportLifecycle = void 0;
|
|
1309
|
+
}
|
|
1310
|
+
handleOnline = () => {
|
|
1311
|
+
this.emitConnectionLost("online");
|
|
1312
|
+
};
|
|
1313
|
+
handleHidden = () => {
|
|
1314
|
+
this.pauseAudioAnalysis();
|
|
1315
|
+
};
|
|
1316
|
+
handleVisible = () => {
|
|
1317
|
+
this.resumeAudioAnalysis();
|
|
1318
|
+
this.emitConnectionLost("visibility");
|
|
1319
|
+
};
|
|
1320
|
+
handlePageHide = () => {
|
|
1321
|
+
this.pauseAudioAnalysis();
|
|
1322
|
+
};
|
|
1323
|
+
handlePageShow = (event) => {
|
|
1324
|
+
this.resumeAudioAnalysis();
|
|
1325
|
+
if (event.persisted) {
|
|
1326
|
+
this.emitConnectionLost("pageshow");
|
|
1327
|
+
}
|
|
1328
|
+
};
|
|
1329
|
+
handleDeviceChange = () => {
|
|
1330
|
+
void this.refreshMicrophoneTrack();
|
|
1331
|
+
};
|
|
1332
|
+
async refreshMicrophoneTrack() {
|
|
1333
|
+
if (!this.pc || this.isExplicitDisconnect) {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
try {
|
|
1337
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
1338
|
+
audioSender: this.audioSender,
|
|
1339
|
+
mediaStream: this.mediaStream,
|
|
1340
|
+
peerConnection: this.pc
|
|
1341
|
+
});
|
|
1342
|
+
this.audioSender = audioSender;
|
|
1343
|
+
this.mediaStream = mediaStream;
|
|
1344
|
+
} catch (error) {
|
|
1345
|
+
this.emitEvent({
|
|
1346
|
+
type: "error",
|
|
1347
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
emitConnectionLost(cause, error) {
|
|
1352
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
|
|
1353
|
+
return;
|
|
1354
|
+
}
|
|
1355
|
+
this.connectionLossNotified = true;
|
|
1356
|
+
this.isResponseInProgress = false;
|
|
1357
|
+
this.cancelInFlight = false;
|
|
1358
|
+
this.resetResponseTracking();
|
|
1359
|
+
this.stopLipSyncAnalysis();
|
|
1360
|
+
this.emitEvent({
|
|
1361
|
+
type: "connection.lost",
|
|
1362
|
+
cause,
|
|
1363
|
+
error
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
needsRecovery(cause) {
|
|
1367
|
+
if (!this.pc || !this.dc) {
|
|
1368
|
+
return false;
|
|
1369
|
+
}
|
|
1370
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
1371
|
+
return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
|
|
1372
|
+
}
|
|
1373
|
+
return true;
|
|
1374
|
+
}
|
|
1375
|
+
prepareAudioAnalysis() {
|
|
1376
|
+
if (this.audioContext) {
|
|
1377
|
+
return;
|
|
1378
|
+
}
|
|
1379
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
1380
|
+
if (!audioContextConstructor) {
|
|
1381
|
+
return;
|
|
1382
|
+
}
|
|
1383
|
+
this.audioContext = new audioContextConstructor();
|
|
1384
|
+
}
|
|
882
1385
|
setupAudioAnalysis(stream) {
|
|
883
1386
|
try {
|
|
884
|
-
|
|
885
|
-
if (!
|
|
886
|
-
|
|
1387
|
+
this.prepareAudioAnalysis();
|
|
1388
|
+
if (!this.audioContext) {
|
|
1389
|
+
return;
|
|
887
1390
|
}
|
|
888
|
-
this.
|
|
889
|
-
|
|
1391
|
+
this.audioSource?.disconnect();
|
|
1392
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
890
1393
|
this.analyser = this.audioContext.createAnalyser();
|
|
891
1394
|
this.analyser.fftSize = 256;
|
|
892
1395
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
893
|
-
|
|
1396
|
+
this.audioSource.connect(this.analyser);
|
|
894
1397
|
this.startLipSyncAnalysis();
|
|
895
1398
|
} catch (error) {
|
|
896
1399
|
console.error("Failed to setup audio analysis:", error);
|
|
@@ -926,8 +1429,23 @@ var OpenAIRealtimeClient = class {
|
|
|
926
1429
|
}
|
|
927
1430
|
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
928
1431
|
}
|
|
1432
|
+
pauseAudioAnalysis() {
|
|
1433
|
+
this.stopLipSyncAnalysis();
|
|
1434
|
+
}
|
|
1435
|
+
resumeAudioAnalysis() {
|
|
1436
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
this.startLipSyncAnalysis();
|
|
1440
|
+
}
|
|
929
1441
|
cleanup() {
|
|
1442
|
+
this.isCleaningUp = true;
|
|
1443
|
+
this.unbindTransportLifecycleEvents();
|
|
1444
|
+
this.iceDisconnectDebouncer.cancel();
|
|
930
1445
|
this.stopLipSyncAnalysis();
|
|
1446
|
+
this.rejectPendingSessionUpdate(
|
|
1447
|
+
new Error("Realtime session ended before session update completed")
|
|
1448
|
+
);
|
|
931
1449
|
if (this.dc) {
|
|
932
1450
|
this.dc.close();
|
|
933
1451
|
this.dc = null;
|
|
@@ -948,9 +1466,13 @@ var OpenAIRealtimeClient = class {
|
|
|
948
1466
|
this.audioContext.close();
|
|
949
1467
|
this.audioContext = null;
|
|
950
1468
|
}
|
|
1469
|
+
this.audioSource = null;
|
|
1470
|
+
this.audioSender = null;
|
|
951
1471
|
this.analyser = null;
|
|
952
1472
|
this.isResponseInProgress = false;
|
|
953
1473
|
this.cancelInFlight = false;
|
|
1474
|
+
this.connectionLossNotified = false;
|
|
1475
|
+
this.isCleaningUp = false;
|
|
954
1476
|
this.resetResponseTracking();
|
|
955
1477
|
}
|
|
956
1478
|
beginResponseRequest() {
|
|
@@ -959,11 +1481,86 @@ var OpenAIRealtimeClient = class {
|
|
|
959
1481
|
this.cancelInFlight = false;
|
|
960
1482
|
}
|
|
961
1483
|
requireOpenDataChannel() {
|
|
1484
|
+
if (this.isRecovering) {
|
|
1485
|
+
throw new Error("Realtime transport reconnecting");
|
|
1486
|
+
}
|
|
962
1487
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
963
1488
|
throw new Error("DataChannel not ready");
|
|
964
1489
|
}
|
|
965
1490
|
return this.dc;
|
|
966
1491
|
}
|
|
1492
|
+
async tryIceRestartRecovery(config) {
|
|
1493
|
+
const pc = this.pc;
|
|
1494
|
+
if (!pc || this.shouldRebuildConnection()) {
|
|
1495
|
+
return false;
|
|
1496
|
+
}
|
|
1497
|
+
if (pc.iceConnectionState === "connected") {
|
|
1498
|
+
return true;
|
|
1499
|
+
}
|
|
1500
|
+
if (pc.iceConnectionState === "disconnected") {
|
|
1501
|
+
await delay(1500);
|
|
1502
|
+
if (pc.iceConnectionState !== "disconnected") {
|
|
1503
|
+
return true;
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
1507
|
+
return false;
|
|
1508
|
+
}
|
|
1509
|
+
if (typeof pc.restartIce === "function") {
|
|
1510
|
+
pc.restartIce();
|
|
1511
|
+
}
|
|
1512
|
+
const offer = await pc.createOffer({ iceRestart: true });
|
|
1513
|
+
await pc.setLocalDescription(offer);
|
|
1514
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
1515
|
+
transport: "webrtc",
|
|
1516
|
+
session: config ?? this.currentSessionConfig ?? {},
|
|
1517
|
+
sdpOffer: offer.sdp
|
|
1518
|
+
});
|
|
1519
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
1520
|
+
throw new Error(
|
|
1521
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1522
|
+
);
|
|
1523
|
+
}
|
|
1524
|
+
await pc.setRemoteDescription({
|
|
1525
|
+
type: "answer",
|
|
1526
|
+
sdp: bootstrap.answerSdp
|
|
1527
|
+
});
|
|
1528
|
+
return true;
|
|
1529
|
+
}
|
|
1530
|
+
shouldRebuildConnection() {
|
|
1531
|
+
return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
|
|
1532
|
+
}
|
|
1533
|
+
async rebuildConnection(config) {
|
|
1534
|
+
this.connectionLossNotified = true;
|
|
1535
|
+
this.cleanup();
|
|
1536
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
1537
|
+
}
|
|
1538
|
+
resolvePendingSessionUpdate() {
|
|
1539
|
+
if (!this.pendingSessionUpdate) {
|
|
1540
|
+
return;
|
|
1541
|
+
}
|
|
1542
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
1543
|
+
this.pendingSessionUpdate.resolve();
|
|
1544
|
+
this.pendingSessionUpdate = null;
|
|
1545
|
+
}
|
|
1546
|
+
rejectPendingSessionUpdate(error) {
|
|
1547
|
+
if (!this.pendingSessionUpdate) {
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1550
|
+
clearTimeout(this.pendingSessionUpdate.timeoutId);
|
|
1551
|
+
this.pendingSessionUpdate.reject(error);
|
|
1552
|
+
this.pendingSessionUpdate = null;
|
|
1553
|
+
}
|
|
1554
|
+
isPendingSessionUpdateError(event) {
|
|
1555
|
+
const pendingUpdate = this.pendingSessionUpdate;
|
|
1556
|
+
if (!pendingUpdate) {
|
|
1557
|
+
return false;
|
|
1558
|
+
}
|
|
1559
|
+
return event.error?.event_id === pendingUpdate.eventId;
|
|
1560
|
+
}
|
|
1561
|
+
getSessionUpdateTimeoutMs() {
|
|
1562
|
+
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
1563
|
+
}
|
|
967
1564
|
resetResponseTracking() {
|
|
968
1565
|
this.assistantText = "";
|
|
969
1566
|
this.hasStartedAssistantResponse = false;
|
|
@@ -989,6 +1586,34 @@ var OpenAIRealtimeClient = class {
|
|
|
989
1586
|
function createOpenAIRealtimeClient(options) {
|
|
990
1587
|
return new OpenAIRealtimeClient(options);
|
|
991
1588
|
}
|
|
1589
|
+
function toOpenAIRealtimeSessionUpdate(config) {
|
|
1590
|
+
const session = {
|
|
1591
|
+
audio: {
|
|
1592
|
+
output: {
|
|
1593
|
+
voice: config?.voice ?? DEFAULT_REALTIME_VOICE
|
|
1594
|
+
}
|
|
1595
|
+
},
|
|
1596
|
+
tool_choice: config?.toolChoice ?? "auto"
|
|
1597
|
+
};
|
|
1598
|
+
if (config?.instructions) {
|
|
1599
|
+
session.instructions = config.instructions;
|
|
1600
|
+
}
|
|
1601
|
+
if (config?.temperature !== void 0) {
|
|
1602
|
+
session.temperature = config.temperature;
|
|
1603
|
+
}
|
|
1604
|
+
if (config?.maxTokens !== void 0) {
|
|
1605
|
+
session.max_response_output_tokens = config.maxTokens;
|
|
1606
|
+
}
|
|
1607
|
+
if (config?.tools?.length) {
|
|
1608
|
+
session.tools = config.tools;
|
|
1609
|
+
}
|
|
1610
|
+
return session;
|
|
1611
|
+
}
|
|
1612
|
+
function delay(ms) {
|
|
1613
|
+
return new Promise((resolve) => {
|
|
1614
|
+
setTimeout(resolve, ms);
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
992
1617
|
|
|
993
1618
|
// src/remote/index.ts
|
|
994
1619
|
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|
|
@@ -1037,6 +1662,12 @@ var RemoteRealtimeClient = class {
|
|
|
1037
1662
|
throw error;
|
|
1038
1663
|
}
|
|
1039
1664
|
}
|
|
1665
|
+
async updateSession(config) {
|
|
1666
|
+
await this.getActiveTransportClient().updateSession(config);
|
|
1667
|
+
}
|
|
1668
|
+
async recover(config) {
|
|
1669
|
+
await this.getActiveTransportClient().recover(config);
|
|
1670
|
+
}
|
|
1040
1671
|
async disconnect() {
|
|
1041
1672
|
if (!this.transportClient) {
|
|
1042
1673
|
return;
|