@charivo/realtime 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -7
- package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
- package/dist/chunk-DHNRDHST.mjs.map +1 -0
- package/dist/chunk-PDODY2T7.mjs +65 -0
- package/dist/chunk-PDODY2T7.mjs.map +1 -0
- package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
- package/dist/chunk-PTFZ7CML.mjs.map +1 -0
- package/dist/chunk-Y7JO2FFX.mjs +121 -0
- package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
- package/dist/index.d.mts +16 -4
- package/dist/index.d.ts +16 -4
- package/dist/index.js +240 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +238 -98
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +37 -1
- package/dist/openai/index.d.ts +37 -1
- package/dist/openai/index.js +427 -14
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.d.mts +26 -1
- package/dist/openai-agents/index.d.ts +26 -1
- package/dist/openai-agents/index.js +311 -18
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -2
- package/dist/remote/index.d.mts +3 -1
- package/dist/remote/index.d.ts +3 -1
- package/dist/remote/index.js +663 -32
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +10 -3
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
- package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
- package/package.json +3 -3
- package/dist/chunk-IIKECRXN.mjs.map +0 -1
- package/dist/chunk-JGJZKKMH.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
|
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
|
|
|
5
5
|
|
|
6
6
|
// src/openai-agents/client.ts
|
|
7
7
|
|
|
8
|
+
// src/internal/microphone.ts
|
|
9
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
10
|
+
echoCancellation: true,
|
|
11
|
+
noiseSuppression: true,
|
|
12
|
+
autoGainControl: true
|
|
13
|
+
};
|
|
14
|
+
async function acquireMicrophoneStream() {
|
|
15
|
+
return navigator.mediaDevices.getUserMedia({
|
|
16
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
20
|
+
function bindTransportLifecycle(callbacks) {
|
|
21
|
+
const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
22
|
+
onHidden: callbacks.onHidden,
|
|
23
|
+
onOnline: callbacks.onOnline,
|
|
24
|
+
onPageHide: callbacks.onPageHide,
|
|
25
|
+
onPageShow: callbacks.onPageShow,
|
|
26
|
+
onVisible: callbacks.onVisible
|
|
27
|
+
});
|
|
28
|
+
if (callbacks.onDeviceChange) {
|
|
29
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
30
|
+
"devicechange",
|
|
31
|
+
callbacks.onDeviceChange
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return () => {
|
|
35
|
+
teardownBrowserLifecycle();
|
|
36
|
+
if (callbacks.onDeviceChange) {
|
|
37
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
38
|
+
"devicechange",
|
|
39
|
+
callbacks.onDeviceChange
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
async function replaceMicrophoneTrack(state) {
|
|
45
|
+
if (!state.peerConnection) {
|
|
46
|
+
throw new Error("Realtime transport not active");
|
|
47
|
+
}
|
|
48
|
+
const nextStream = await acquireMicrophoneStream();
|
|
49
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
50
|
+
if (!nextTrack) {
|
|
51
|
+
throw new Error("Microphone access required for Realtime API");
|
|
52
|
+
}
|
|
53
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
54
|
+
if (!sender) {
|
|
55
|
+
throw new Error("No outbound audio sender available");
|
|
56
|
+
}
|
|
57
|
+
await sender.replaceTrack(nextTrack);
|
|
58
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
59
|
+
return {
|
|
60
|
+
audioSender: sender,
|
|
61
|
+
mediaStream: nextStream
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
65
|
+
let pending = null;
|
|
66
|
+
return {
|
|
67
|
+
schedule() {
|
|
68
|
+
if (pending) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
pending = setTimeout(() => {
|
|
72
|
+
pending = null;
|
|
73
|
+
onDisconnect();
|
|
74
|
+
}, delayMs);
|
|
75
|
+
},
|
|
76
|
+
cancel() {
|
|
77
|
+
if (!pending) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
clearTimeout(pending);
|
|
81
|
+
pending = null;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
8
86
|
// src/internal/shared.ts
|
|
9
87
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
10
88
|
function isRecord(value) {
|
|
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
|
|
|
83
161
|
}
|
|
84
162
|
audioAnalysisStream = null;
|
|
85
163
|
audioContext = null;
|
|
164
|
+
audioSource = null;
|
|
86
165
|
analyser = null;
|
|
87
166
|
lipSyncInterval = null;
|
|
88
167
|
pendingAudioElementPoll = null;
|
|
89
168
|
observedAudioElement = null;
|
|
90
169
|
observedAudioElementListener = null;
|
|
170
|
+
async prepareAudioContext() {
|
|
171
|
+
if (this.audioContext) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
175
|
+
if (!audioContextConstructor) {
|
|
176
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
177
|
+
}
|
|
178
|
+
this.audioContext = new audioContextConstructor();
|
|
179
|
+
}
|
|
91
180
|
observeAudioElement(audioElement) {
|
|
92
181
|
this.detachObservedAudioElement();
|
|
93
182
|
const tryAttachStream = () => {
|
|
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
|
|
|
109
198
|
this.stopLipSyncAnalysis();
|
|
110
199
|
this.audioAnalysisStream = stream;
|
|
111
200
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
201
|
+
if (!this.audioContext) {
|
|
202
|
+
void this.prepareAudioContext().catch((error) => {
|
|
203
|
+
this.options.onError?.(error);
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (!this.audioContext) {
|
|
207
|
+
return;
|
|
115
208
|
}
|
|
116
|
-
this.
|
|
117
|
-
|
|
209
|
+
this.audioSource?.disconnect?.();
|
|
210
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
118
211
|
this.analyser = this.audioContext.createAnalyser();
|
|
119
212
|
this.analyser.fftSize = 256;
|
|
120
213
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
121
|
-
|
|
214
|
+
this.audioSource.connect(this.analyser);
|
|
122
215
|
this.startLipSyncAnalysis();
|
|
123
216
|
} catch (error) {
|
|
124
217
|
this.options.onError?.(error);
|
|
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
|
|
|
127
220
|
stopOutput() {
|
|
128
221
|
this.stopLipSyncAnalysis();
|
|
129
222
|
}
|
|
223
|
+
pause() {
|
|
224
|
+
this.stopLipSyncAnalysis();
|
|
225
|
+
}
|
|
226
|
+
resume() {
|
|
227
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
this.startLipSyncAnalysis();
|
|
231
|
+
}
|
|
130
232
|
cleanup() {
|
|
131
233
|
this.stopLipSyncAnalysis();
|
|
132
234
|
this.stopAudioElementPolling();
|
|
133
235
|
this.detachObservedAudioElement();
|
|
236
|
+
this.audioSource?.disconnect?.();
|
|
237
|
+
this.audioSource = null;
|
|
134
238
|
if (this.audioContext) {
|
|
135
239
|
void this.audioContext.close();
|
|
136
240
|
this.audioContext = null;
|
|
@@ -183,18 +287,20 @@ var LipSyncAnalyzer = class {
|
|
|
183
287
|
}
|
|
184
288
|
};
|
|
185
289
|
|
|
290
|
+
// src/instructions.ts
|
|
291
|
+
var DEFAULT_REALTIME_VOICE = "marin";
|
|
292
|
+
|
|
186
293
|
// src/openai-agents/session-config.ts
|
|
187
294
|
var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
|
|
188
295
|
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
189
|
-
var DEFAULT_VOICE = "marin";
|
|
190
296
|
function resolveInstructions(config) {
|
|
191
297
|
return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
|
|
192
298
|
}
|
|
193
299
|
function resolveVoice(config) {
|
|
194
|
-
return config?.voice ??
|
|
300
|
+
return config?.voice ?? DEFAULT_REALTIME_VOICE;
|
|
195
301
|
}
|
|
196
302
|
function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
197
|
-
|
|
303
|
+
const sessionConfig = {
|
|
198
304
|
model: config?.model ?? DEFAULT_MODEL,
|
|
199
305
|
instructions: resolveInstructions(config),
|
|
200
306
|
toolChoice: config?.toolChoice ?? "auto",
|
|
@@ -205,6 +311,13 @@ function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
|
205
311
|
}
|
|
206
312
|
}
|
|
207
313
|
};
|
|
314
|
+
if (config?.temperature !== void 0) {
|
|
315
|
+
sessionConfig.temperature = config.temperature;
|
|
316
|
+
}
|
|
317
|
+
if (config?.maxTokens !== void 0) {
|
|
318
|
+
sessionConfig.maxResponseOutputTokens = config.maxTokens;
|
|
319
|
+
}
|
|
320
|
+
return sessionConfig;
|
|
208
321
|
}
|
|
209
322
|
|
|
210
323
|
// src/openai-agents/tool-schema.ts
|
|
@@ -256,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
256
369
|
session = null;
|
|
257
370
|
transport = null;
|
|
258
371
|
audioElement = null;
|
|
372
|
+
mediaStream = null;
|
|
373
|
+
peerConnection = null;
|
|
374
|
+
audioSender = null;
|
|
259
375
|
connectionWasActive = false;
|
|
376
|
+
connectionLossNotified = false;
|
|
377
|
+
isExplicitDisconnect = false;
|
|
378
|
+
isRecovering = false;
|
|
379
|
+
isCleaningUp = false;
|
|
260
380
|
assistant = { text: "", started: false };
|
|
261
381
|
latestHistory = [];
|
|
382
|
+
currentSessionConfig;
|
|
383
|
+
teardownTransportLifecycle;
|
|
384
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
385
|
+
this.emitConnectionLost("ice-disconnected");
|
|
386
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
262
387
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
263
388
|
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
264
389
|
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
@@ -271,18 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
271
396
|
});
|
|
272
397
|
async connect(config) {
|
|
273
398
|
try {
|
|
399
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
400
|
+
throw new Error("Realtime session disconnect requested");
|
|
401
|
+
}
|
|
274
402
|
this.log("Starting OpenAI Agents Realtime WebRTC connection");
|
|
403
|
+
this.isExplicitDisconnect = false;
|
|
404
|
+
this.isRecovering = false;
|
|
405
|
+
this.connectionLossNotified = false;
|
|
406
|
+
this.currentSessionConfig = config;
|
|
275
407
|
this.audioElement = document.createElement("audio");
|
|
276
408
|
this.audioElement.autoplay = true;
|
|
277
409
|
this.audioElement.setAttribute("playsinline", "true");
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
voice: resolveVoice(config)
|
|
283
|
-
});
|
|
410
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
411
|
+
this.bindTransportLifecycleEvents();
|
|
412
|
+
await this.lipSyncAnalyzer.prepareAudioContext();
|
|
413
|
+
const agent = this.createAgent(config);
|
|
284
414
|
this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
|
|
285
|
-
this.createTransportOptions(this.audioElement)
|
|
415
|
+
this.createTransportOptions(this.audioElement, this.mediaStream)
|
|
286
416
|
);
|
|
287
417
|
this.session = new agentsRealtime.RealtimeSession(agent, {
|
|
288
418
|
transport: this.transport,
|
|
@@ -314,9 +444,37 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
314
444
|
}
|
|
315
445
|
async disconnect() {
|
|
316
446
|
this.log("Disconnecting OpenAI Agents Realtime WebRTC");
|
|
447
|
+
this.isExplicitDisconnect = true;
|
|
448
|
+
this.isRecovering = false;
|
|
317
449
|
this.cleanup();
|
|
318
450
|
}
|
|
451
|
+
async updateSession(config) {
|
|
452
|
+
if (!this.session) {
|
|
453
|
+
throw new Error("Realtime session not active");
|
|
454
|
+
}
|
|
455
|
+
this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
|
|
456
|
+
await this.session.updateAgent(this.createAgent(config));
|
|
457
|
+
this.currentSessionConfig = config;
|
|
458
|
+
}
|
|
459
|
+
async recover(config) {
|
|
460
|
+
this.isRecovering = true;
|
|
461
|
+
this.connectionLossNotified = true;
|
|
462
|
+
this.currentSessionConfig = config;
|
|
463
|
+
try {
|
|
464
|
+
this.cleanupPendingToolCalls(
|
|
465
|
+
new Error("Realtime session interrupted during reconnect")
|
|
466
|
+
);
|
|
467
|
+
this.cleanup();
|
|
468
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
469
|
+
this.connectionLossNotified = false;
|
|
470
|
+
} finally {
|
|
471
|
+
this.isRecovering = false;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
319
474
|
async sendText(text) {
|
|
475
|
+
if (this.isRecovering) {
|
|
476
|
+
throw new Error("Realtime transport reconnecting");
|
|
477
|
+
}
|
|
320
478
|
if (!this.session) {
|
|
321
479
|
throw new Error("Realtime session not active");
|
|
322
480
|
}
|
|
@@ -324,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
324
482
|
this.session.sendMessage(text);
|
|
325
483
|
}
|
|
326
484
|
async sendAudio(_audio) {
|
|
485
|
+
if (this.isRecovering) {
|
|
486
|
+
throw new Error("Realtime transport reconnecting");
|
|
487
|
+
}
|
|
327
488
|
console.warn(
|
|
328
489
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
329
490
|
);
|
|
330
491
|
}
|
|
331
492
|
async sendToolResult(callId, output) {
|
|
493
|
+
if (this.isRecovering) {
|
|
494
|
+
throw new Error("Realtime transport reconnecting");
|
|
495
|
+
}
|
|
332
496
|
const pendingCall = this.pendingToolCalls.get(callId);
|
|
333
497
|
if (!pendingCall) {
|
|
334
498
|
throw new Error(`No pending realtime tool call for "${callId}"`);
|
|
@@ -337,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
337
501
|
pendingCall.resolve(output);
|
|
338
502
|
}
|
|
339
503
|
async interrupt() {
|
|
504
|
+
if (this.isRecovering) {
|
|
505
|
+
throw new Error("Realtime transport reconnecting");
|
|
506
|
+
}
|
|
340
507
|
if (!this.session) {
|
|
341
508
|
throw new Error("Realtime session not active");
|
|
342
509
|
}
|
|
@@ -346,6 +513,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
346
513
|
onEvent(callback) {
|
|
347
514
|
this.eventCallbacks.add(callback);
|
|
348
515
|
}
|
|
516
|
+
createAgent(config) {
|
|
517
|
+
return new agentsRealtime.RealtimeAgent({
|
|
518
|
+
name: "charivo-realtime-agent",
|
|
519
|
+
instructions: resolveInstructions(config),
|
|
520
|
+
tools: this.createProxyTools(config?.tools),
|
|
521
|
+
voice: resolveVoice(config)
|
|
522
|
+
});
|
|
523
|
+
}
|
|
349
524
|
bindSessionEvents(session) {
|
|
350
525
|
session.on("agent_start", () => {
|
|
351
526
|
this.ensureAssistantStarted();
|
|
@@ -392,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
392
567
|
if (status === "disconnected" && this.connectionWasActive) {
|
|
393
568
|
this.cleanupPendingToolCalls();
|
|
394
569
|
this.connectionWasActive = false;
|
|
395
|
-
this.
|
|
570
|
+
this.emitConnectionLost("connection-failed");
|
|
396
571
|
}
|
|
397
572
|
});
|
|
398
573
|
}
|
|
@@ -494,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
494
669
|
});
|
|
495
670
|
});
|
|
496
671
|
}
|
|
497
|
-
createTransportOptions(audioElement) {
|
|
672
|
+
createTransportOptions(audioElement, mediaStream) {
|
|
498
673
|
return {
|
|
499
674
|
audioElement,
|
|
675
|
+
mediaStream,
|
|
500
676
|
changePeerConnection: async (peerConnection) => {
|
|
677
|
+
this.peerConnection = peerConnection;
|
|
678
|
+
this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
|
|
679
|
+
this.bindPeerConnectionEvents(peerConnection);
|
|
501
680
|
peerConnection.addEventListener("track", (event) => {
|
|
502
681
|
const stream = event.streams[0];
|
|
503
682
|
if (stream) {
|
|
@@ -508,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
508
687
|
}
|
|
509
688
|
};
|
|
510
689
|
}
|
|
690
|
+
bindPeerConnectionEvents(peerConnection) {
|
|
691
|
+
peerConnection.addEventListener("iceconnectionstatechange", () => {
|
|
692
|
+
const iceState = peerConnection.iceConnectionState;
|
|
693
|
+
if (iceState === "failed") {
|
|
694
|
+
this.iceDisconnectDebouncer.cancel();
|
|
695
|
+
this.emitConnectionLost("ice-failed");
|
|
696
|
+
} else if (iceState === "disconnected") {
|
|
697
|
+
this.iceDisconnectDebouncer.schedule();
|
|
698
|
+
} else {
|
|
699
|
+
this.iceDisconnectDebouncer.cancel();
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
peerConnection.addEventListener("connectionstatechange", () => {
|
|
703
|
+
if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
|
|
704
|
+
this.iceDisconnectDebouncer.cancel();
|
|
705
|
+
this.emitConnectionLost("connection-failed");
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
bindTransportLifecycleEvents() {
|
|
710
|
+
if (this.teardownTransportLifecycle) {
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
714
|
+
onHidden: this.handleHidden,
|
|
715
|
+
onOnline: this.handleOnline,
|
|
716
|
+
onPageHide: this.handlePageHide,
|
|
717
|
+
onPageShow: this.handlePageShow,
|
|
718
|
+
onVisible: this.handleVisible,
|
|
719
|
+
onDeviceChange: this.handleDeviceChange
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
unbindTransportLifecycleEvents() {
|
|
723
|
+
this.teardownTransportLifecycle?.();
|
|
724
|
+
this.teardownTransportLifecycle = void 0;
|
|
725
|
+
}
|
|
726
|
+
handleOnline = () => {
|
|
727
|
+
this.emitConnectionLost("online");
|
|
728
|
+
};
|
|
729
|
+
handleHidden = () => {
|
|
730
|
+
this.lipSyncAnalyzer.pause();
|
|
731
|
+
};
|
|
732
|
+
handleVisible = () => {
|
|
733
|
+
this.lipSyncAnalyzer.resume();
|
|
734
|
+
this.emitConnectionLost("visibility");
|
|
735
|
+
};
|
|
736
|
+
handlePageHide = () => {
|
|
737
|
+
this.lipSyncAnalyzer.pause();
|
|
738
|
+
};
|
|
739
|
+
handlePageShow = (event) => {
|
|
740
|
+
this.lipSyncAnalyzer.resume();
|
|
741
|
+
if (event.persisted) {
|
|
742
|
+
this.emitConnectionLost("pageshow");
|
|
743
|
+
}
|
|
744
|
+
};
|
|
745
|
+
handleDeviceChange = () => {
|
|
746
|
+
void this.refreshMicrophoneTrack();
|
|
747
|
+
};
|
|
748
|
+
async refreshMicrophoneTrack() {
|
|
749
|
+
if (!this.peerConnection || this.isExplicitDisconnect) {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
try {
|
|
753
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
754
|
+
audioSender: this.audioSender,
|
|
755
|
+
mediaStream: this.mediaStream,
|
|
756
|
+
peerConnection: this.peerConnection
|
|
757
|
+
});
|
|
758
|
+
this.audioSender = audioSender;
|
|
759
|
+
this.mediaStream = mediaStream;
|
|
760
|
+
} catch (error) {
|
|
761
|
+
this.emitEvent({
|
|
762
|
+
type: "error",
|
|
763
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
emitConnectionLost(cause, error) {
|
|
768
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
this.connectionLossNotified = true;
|
|
772
|
+
this.cleanupPendingToolCalls(
|
|
773
|
+
new Error("Realtime session interrupted during reconnect")
|
|
774
|
+
);
|
|
775
|
+
this.resetAssistantTracking();
|
|
776
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
777
|
+
this.emitEvent({
|
|
778
|
+
type: "connection.lost",
|
|
779
|
+
cause,
|
|
780
|
+
error
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
shouldRecover(cause) {
|
|
784
|
+
if (!this.peerConnection) {
|
|
785
|
+
return false;
|
|
786
|
+
}
|
|
787
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
788
|
+
return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
|
|
789
|
+
}
|
|
790
|
+
return true;
|
|
791
|
+
}
|
|
511
792
|
cleanup(error) {
|
|
793
|
+
this.connectionWasActive = false;
|
|
794
|
+
this.isCleaningUp = true;
|
|
795
|
+
this.iceDisconnectDebouncer.cancel();
|
|
796
|
+
this.unbindTransportLifecycleEvents();
|
|
512
797
|
this.cleanupPendingToolCalls(error);
|
|
513
798
|
this.lipSyncAnalyzer.cleanup();
|
|
514
799
|
if (this.transport) {
|
|
@@ -523,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
523
808
|
this.audioElement.srcObject = null;
|
|
524
809
|
this.audioElement = null;
|
|
525
810
|
}
|
|
811
|
+
if (this.mediaStream) {
|
|
812
|
+
this.mediaStream.getTracks?.().forEach((track) => track.stop());
|
|
813
|
+
this.mediaStream = null;
|
|
814
|
+
}
|
|
815
|
+
this.peerConnection = null;
|
|
816
|
+
this.audioSender = null;
|
|
526
817
|
this.latestHistory = [];
|
|
818
|
+
this.connectionLossNotified = false;
|
|
819
|
+
this.isCleaningUp = false;
|
|
527
820
|
this.resetAssistantTracking();
|
|
528
821
|
}
|
|
529
822
|
resetAssistantTracking() {
|