@charivo/realtime 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/{chunk-WLTIMNHQ.mjs → chunk-DHNRDHST.mjs} +248 -13
- package/dist/chunk-DHNRDHST.mjs.map +1 -0
- package/dist/{chunk-FABRM56W.mjs → chunk-PTFZ7CML.mjs} +208 -12
- package/dist/chunk-PTFZ7CML.mjs.map +1 -0
- package/dist/chunk-Y7JO2FFX.mjs +121 -0
- package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +217 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +214 -3
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +29 -1
- package/dist/openai/index.d.ts +29 -1
- package/dist/openai/index.js +323 -10
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +2 -2
- package/dist/openai-agents/index.d.mts +24 -1
- package/dist/openai-agents/index.d.ts +24 -1
- package/dist/openai-agents/index.js +283 -9
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +2 -2
- package/dist/remote/index.d.mts +2 -1
- package/dist/remote/index.d.ts +2 -1
- package/dist/remote/index.js +531 -19
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +6 -3
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
- package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
- package/package.json +2 -2
- package/dist/chunk-FABRM56W.mjs.map +0 -1
- package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/remote/index.js
CHANGED
|
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
|
|
|
5
5
|
|
|
6
6
|
// src/remote/index.ts
|
|
7
7
|
|
|
8
|
+
// src/internal/microphone.ts
|
|
9
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
10
|
+
echoCancellation: true,
|
|
11
|
+
noiseSuppression: true,
|
|
12
|
+
autoGainControl: true
|
|
13
|
+
};
|
|
14
|
+
async function acquireMicrophoneStream() {
|
|
15
|
+
return navigator.mediaDevices.getUserMedia({
|
|
16
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
20
|
+
function bindTransportLifecycle(callbacks) {
|
|
21
|
+
const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
22
|
+
onHidden: callbacks.onHidden,
|
|
23
|
+
onOnline: callbacks.onOnline,
|
|
24
|
+
onPageHide: callbacks.onPageHide,
|
|
25
|
+
onPageShow: callbacks.onPageShow,
|
|
26
|
+
onVisible: callbacks.onVisible
|
|
27
|
+
});
|
|
28
|
+
if (callbacks.onDeviceChange) {
|
|
29
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
30
|
+
"devicechange",
|
|
31
|
+
callbacks.onDeviceChange
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return () => {
|
|
35
|
+
teardownBrowserLifecycle();
|
|
36
|
+
if (callbacks.onDeviceChange) {
|
|
37
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
38
|
+
"devicechange",
|
|
39
|
+
callbacks.onDeviceChange
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
async function replaceMicrophoneTrack(state) {
|
|
45
|
+
if (!state.peerConnection) {
|
|
46
|
+
throw new Error("Realtime transport not active");
|
|
47
|
+
}
|
|
48
|
+
const nextStream = await acquireMicrophoneStream();
|
|
49
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
50
|
+
if (!nextTrack) {
|
|
51
|
+
throw new Error("Microphone access required for Realtime API");
|
|
52
|
+
}
|
|
53
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
54
|
+
if (!sender) {
|
|
55
|
+
throw new Error("No outbound audio sender available");
|
|
56
|
+
}
|
|
57
|
+
await sender.replaceTrack(nextTrack);
|
|
58
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
59
|
+
return {
|
|
60
|
+
audioSender: sender,
|
|
61
|
+
mediaStream: nextStream
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
65
|
+
let pending = null;
|
|
66
|
+
return {
|
|
67
|
+
schedule() {
|
|
68
|
+
if (pending) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
pending = setTimeout(() => {
|
|
72
|
+
pending = null;
|
|
73
|
+
onDisconnect();
|
|
74
|
+
}, delayMs);
|
|
75
|
+
},
|
|
76
|
+
cancel() {
|
|
77
|
+
if (!pending) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
clearTimeout(pending);
|
|
81
|
+
pending = null;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
8
86
|
// src/internal/shared.ts
|
|
9
87
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
10
88
|
function isRecord(value) {
|
|
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
|
|
|
83
161
|
}
|
|
84
162
|
audioAnalysisStream = null;
|
|
85
163
|
audioContext = null;
|
|
164
|
+
audioSource = null;
|
|
86
165
|
analyser = null;
|
|
87
166
|
lipSyncInterval = null;
|
|
88
167
|
pendingAudioElementPoll = null;
|
|
89
168
|
observedAudioElement = null;
|
|
90
169
|
observedAudioElementListener = null;
|
|
170
|
+
async prepareAudioContext() {
|
|
171
|
+
if (this.audioContext) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
175
|
+
if (!audioContextConstructor) {
|
|
176
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
177
|
+
}
|
|
178
|
+
this.audioContext = new audioContextConstructor();
|
|
179
|
+
}
|
|
91
180
|
observeAudioElement(audioElement) {
|
|
92
181
|
this.detachObservedAudioElement();
|
|
93
182
|
const tryAttachStream = () => {
|
|
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
|
|
|
109
198
|
this.stopLipSyncAnalysis();
|
|
110
199
|
this.audioAnalysisStream = stream;
|
|
111
200
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
201
|
+
if (!this.audioContext) {
|
|
202
|
+
void this.prepareAudioContext().catch((error) => {
|
|
203
|
+
this.options.onError?.(error);
|
|
204
|
+
});
|
|
115
205
|
}
|
|
116
|
-
this.audioContext
|
|
117
|
-
|
|
206
|
+
if (!this.audioContext) {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
this.audioSource?.disconnect?.();
|
|
210
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
118
211
|
this.analyser = this.audioContext.createAnalyser();
|
|
119
212
|
this.analyser.fftSize = 256;
|
|
120
213
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
121
|
-
|
|
214
|
+
this.audioSource.connect(this.analyser);
|
|
122
215
|
this.startLipSyncAnalysis();
|
|
123
216
|
} catch (error) {
|
|
124
217
|
this.options.onError?.(error);
|
|
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
|
|
|
127
220
|
stopOutput() {
|
|
128
221
|
this.stopLipSyncAnalysis();
|
|
129
222
|
}
|
|
223
|
+
pause() {
|
|
224
|
+
this.stopLipSyncAnalysis();
|
|
225
|
+
}
|
|
226
|
+
resume() {
|
|
227
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
this.startLipSyncAnalysis();
|
|
231
|
+
}
|
|
130
232
|
cleanup() {
|
|
131
233
|
this.stopLipSyncAnalysis();
|
|
132
234
|
this.stopAudioElementPolling();
|
|
133
235
|
this.detachObservedAudioElement();
|
|
236
|
+
this.audioSource?.disconnect?.();
|
|
237
|
+
this.audioSource = null;
|
|
134
238
|
if (this.audioContext) {
|
|
135
239
|
void this.audioContext.close();
|
|
136
240
|
this.audioContext = null;
|
|
@@ -265,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
265
369
|
session = null;
|
|
266
370
|
transport = null;
|
|
267
371
|
audioElement = null;
|
|
372
|
+
mediaStream = null;
|
|
373
|
+
peerConnection = null;
|
|
374
|
+
audioSender = null;
|
|
268
375
|
connectionWasActive = false;
|
|
376
|
+
connectionLossNotified = false;
|
|
377
|
+
isExplicitDisconnect = false;
|
|
378
|
+
isRecovering = false;
|
|
379
|
+
isCleaningUp = false;
|
|
269
380
|
assistant = { text: "", started: false };
|
|
270
381
|
latestHistory = [];
|
|
382
|
+
currentSessionConfig;
|
|
383
|
+
teardownTransportLifecycle;
|
|
384
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
385
|
+
this.emitConnectionLost("ice-disconnected");
|
|
386
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
271
387
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
272
388
|
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
273
389
|
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
@@ -280,13 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
280
396
|
});
|
|
281
397
|
async connect(config) {
|
|
282
398
|
try {
|
|
399
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
400
|
+
throw new Error("Realtime session disconnect requested");
|
|
401
|
+
}
|
|
283
402
|
this.log("Starting OpenAI Agents Realtime WebRTC connection");
|
|
403
|
+
this.isExplicitDisconnect = false;
|
|
404
|
+
this.isRecovering = false;
|
|
405
|
+
this.connectionLossNotified = false;
|
|
406
|
+
this.currentSessionConfig = config;
|
|
284
407
|
this.audioElement = document.createElement("audio");
|
|
285
408
|
this.audioElement.autoplay = true;
|
|
286
409
|
this.audioElement.setAttribute("playsinline", "true");
|
|
410
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
411
|
+
this.bindTransportLifecycleEvents();
|
|
412
|
+
await this.lipSyncAnalyzer.prepareAudioContext();
|
|
287
413
|
const agent = this.createAgent(config);
|
|
288
414
|
this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
|
|
289
|
-
this.createTransportOptions(this.audioElement)
|
|
415
|
+
this.createTransportOptions(this.audioElement, this.mediaStream)
|
|
290
416
|
);
|
|
291
417
|
this.session = new agentsRealtime.RealtimeSession(agent, {
|
|
292
418
|
transport: this.transport,
|
|
@@ -318,6 +444,8 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
318
444
|
}
|
|
319
445
|
async disconnect() {
|
|
320
446
|
this.log("Disconnecting OpenAI Agents Realtime WebRTC");
|
|
447
|
+
this.isExplicitDisconnect = true;
|
|
448
|
+
this.isRecovering = false;
|
|
321
449
|
this.cleanup();
|
|
322
450
|
}
|
|
323
451
|
async updateSession(config) {
|
|
@@ -326,8 +454,27 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
326
454
|
}
|
|
327
455
|
this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
|
|
328
456
|
await this.session.updateAgent(this.createAgent(config));
|
|
457
|
+
this.currentSessionConfig = config;
|
|
458
|
+
}
|
|
459
|
+
async recover(config) {
|
|
460
|
+
this.isRecovering = true;
|
|
461
|
+
this.connectionLossNotified = true;
|
|
462
|
+
this.currentSessionConfig = config;
|
|
463
|
+
try {
|
|
464
|
+
this.cleanupPendingToolCalls(
|
|
465
|
+
new Error("Realtime session interrupted during reconnect")
|
|
466
|
+
);
|
|
467
|
+
this.cleanup();
|
|
468
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
469
|
+
this.connectionLossNotified = false;
|
|
470
|
+
} finally {
|
|
471
|
+
this.isRecovering = false;
|
|
472
|
+
}
|
|
329
473
|
}
|
|
330
474
|
async sendText(text) {
|
|
475
|
+
if (this.isRecovering) {
|
|
476
|
+
throw new Error("Realtime transport reconnecting");
|
|
477
|
+
}
|
|
331
478
|
if (!this.session) {
|
|
332
479
|
throw new Error("Realtime session not active");
|
|
333
480
|
}
|
|
@@ -335,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
335
482
|
this.session.sendMessage(text);
|
|
336
483
|
}
|
|
337
484
|
async sendAudio(_audio) {
|
|
485
|
+
if (this.isRecovering) {
|
|
486
|
+
throw new Error("Realtime transport reconnecting");
|
|
487
|
+
}
|
|
338
488
|
console.warn(
|
|
339
489
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
340
490
|
);
|
|
341
491
|
}
|
|
342
492
|
async sendToolResult(callId, output) {
|
|
493
|
+
if (this.isRecovering) {
|
|
494
|
+
throw new Error("Realtime transport reconnecting");
|
|
495
|
+
}
|
|
343
496
|
const pendingCall = this.pendingToolCalls.get(callId);
|
|
344
497
|
if (!pendingCall) {
|
|
345
498
|
throw new Error(`No pending realtime tool call for "${callId}"`);
|
|
@@ -348,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
348
501
|
pendingCall.resolve(output);
|
|
349
502
|
}
|
|
350
503
|
async interrupt() {
|
|
504
|
+
if (this.isRecovering) {
|
|
505
|
+
throw new Error("Realtime transport reconnecting");
|
|
506
|
+
}
|
|
351
507
|
if (!this.session) {
|
|
352
508
|
throw new Error("Realtime session not active");
|
|
353
509
|
}
|
|
@@ -411,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
411
567
|
if (status === "disconnected" && this.connectionWasActive) {
|
|
412
568
|
this.cleanupPendingToolCalls();
|
|
413
569
|
this.connectionWasActive = false;
|
|
414
|
-
this.
|
|
570
|
+
this.emitConnectionLost("connection-failed");
|
|
415
571
|
}
|
|
416
572
|
});
|
|
417
573
|
}
|
|
@@ -513,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
513
669
|
});
|
|
514
670
|
});
|
|
515
671
|
}
|
|
516
|
-
createTransportOptions(audioElement) {
|
|
672
|
+
createTransportOptions(audioElement, mediaStream) {
|
|
517
673
|
return {
|
|
518
674
|
audioElement,
|
|
675
|
+
mediaStream,
|
|
519
676
|
changePeerConnection: async (peerConnection) => {
|
|
677
|
+
this.peerConnection = peerConnection;
|
|
678
|
+
this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
|
|
679
|
+
this.bindPeerConnectionEvents(peerConnection);
|
|
520
680
|
peerConnection.addEventListener("track", (event) => {
|
|
521
681
|
const stream = event.streams[0];
|
|
522
682
|
if (stream) {
|
|
@@ -527,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
527
687
|
}
|
|
528
688
|
};
|
|
529
689
|
}
|
|
690
|
+
bindPeerConnectionEvents(peerConnection) {
|
|
691
|
+
peerConnection.addEventListener("iceconnectionstatechange", () => {
|
|
692
|
+
const iceState = peerConnection.iceConnectionState;
|
|
693
|
+
if (iceState === "failed") {
|
|
694
|
+
this.iceDisconnectDebouncer.cancel();
|
|
695
|
+
this.emitConnectionLost("ice-failed");
|
|
696
|
+
} else if (iceState === "disconnected") {
|
|
697
|
+
this.iceDisconnectDebouncer.schedule();
|
|
698
|
+
} else {
|
|
699
|
+
this.iceDisconnectDebouncer.cancel();
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
peerConnection.addEventListener("connectionstatechange", () => {
|
|
703
|
+
if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
|
|
704
|
+
this.iceDisconnectDebouncer.cancel();
|
|
705
|
+
this.emitConnectionLost("connection-failed");
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
bindTransportLifecycleEvents() {
|
|
710
|
+
if (this.teardownTransportLifecycle) {
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
714
|
+
onHidden: this.handleHidden,
|
|
715
|
+
onOnline: this.handleOnline,
|
|
716
|
+
onPageHide: this.handlePageHide,
|
|
717
|
+
onPageShow: this.handlePageShow,
|
|
718
|
+
onVisible: this.handleVisible,
|
|
719
|
+
onDeviceChange: this.handleDeviceChange
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
unbindTransportLifecycleEvents() {
|
|
723
|
+
this.teardownTransportLifecycle?.();
|
|
724
|
+
this.teardownTransportLifecycle = void 0;
|
|
725
|
+
}
|
|
726
|
+
handleOnline = () => {
|
|
727
|
+
this.emitConnectionLost("online");
|
|
728
|
+
};
|
|
729
|
+
handleHidden = () => {
|
|
730
|
+
this.lipSyncAnalyzer.pause();
|
|
731
|
+
};
|
|
732
|
+
handleVisible = () => {
|
|
733
|
+
this.lipSyncAnalyzer.resume();
|
|
734
|
+
this.emitConnectionLost("visibility");
|
|
735
|
+
};
|
|
736
|
+
handlePageHide = () => {
|
|
737
|
+
this.lipSyncAnalyzer.pause();
|
|
738
|
+
};
|
|
739
|
+
handlePageShow = (event) => {
|
|
740
|
+
this.lipSyncAnalyzer.resume();
|
|
741
|
+
if (event.persisted) {
|
|
742
|
+
this.emitConnectionLost("pageshow");
|
|
743
|
+
}
|
|
744
|
+
};
|
|
745
|
+
handleDeviceChange = () => {
|
|
746
|
+
void this.refreshMicrophoneTrack();
|
|
747
|
+
};
|
|
748
|
+
async refreshMicrophoneTrack() {
|
|
749
|
+
if (!this.peerConnection || this.isExplicitDisconnect) {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
try {
|
|
753
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
754
|
+
audioSender: this.audioSender,
|
|
755
|
+
mediaStream: this.mediaStream,
|
|
756
|
+
peerConnection: this.peerConnection
|
|
757
|
+
});
|
|
758
|
+
this.audioSender = audioSender;
|
|
759
|
+
this.mediaStream = mediaStream;
|
|
760
|
+
} catch (error) {
|
|
761
|
+
this.emitEvent({
|
|
762
|
+
type: "error",
|
|
763
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
emitConnectionLost(cause, error) {
|
|
768
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
this.connectionLossNotified = true;
|
|
772
|
+
this.cleanupPendingToolCalls(
|
|
773
|
+
new Error("Realtime session interrupted during reconnect")
|
|
774
|
+
);
|
|
775
|
+
this.resetAssistantTracking();
|
|
776
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
777
|
+
this.emitEvent({
|
|
778
|
+
type: "connection.lost",
|
|
779
|
+
cause,
|
|
780
|
+
error
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
shouldRecover(cause) {
|
|
784
|
+
if (!this.peerConnection) {
|
|
785
|
+
return false;
|
|
786
|
+
}
|
|
787
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
788
|
+
return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
|
|
789
|
+
}
|
|
790
|
+
return true;
|
|
791
|
+
}
|
|
530
792
|
cleanup(error) {
|
|
793
|
+
this.connectionWasActive = false;
|
|
794
|
+
this.isCleaningUp = true;
|
|
795
|
+
this.iceDisconnectDebouncer.cancel();
|
|
796
|
+
this.unbindTransportLifecycleEvents();
|
|
531
797
|
this.cleanupPendingToolCalls(error);
|
|
532
798
|
this.lipSyncAnalyzer.cleanup();
|
|
533
799
|
if (this.transport) {
|
|
@@ -542,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
542
808
|
this.audioElement.srcObject = null;
|
|
543
809
|
this.audioElement = null;
|
|
544
810
|
}
|
|
811
|
+
if (this.mediaStream) {
|
|
812
|
+
this.mediaStream.getTracks?.().forEach((track) => track.stop());
|
|
813
|
+
this.mediaStream = null;
|
|
814
|
+
}
|
|
815
|
+
this.peerConnection = null;
|
|
816
|
+
this.audioSender = null;
|
|
545
817
|
this.latestHistory = [];
|
|
818
|
+
this.connectionLossNotified = false;
|
|
819
|
+
this.isCleaningUp = false;
|
|
546
820
|
this.resetAssistantTracking();
|
|
547
821
|
}
|
|
548
822
|
resetAssistantTracking() {
|
|
@@ -597,7 +871,9 @@ var OpenAIRealtimeClient = class {
|
|
|
597
871
|
dc = null;
|
|
598
872
|
audioElement = null;
|
|
599
873
|
mediaStream = null;
|
|
874
|
+
audioSender = null;
|
|
600
875
|
audioContext = null;
|
|
876
|
+
audioSource = null;
|
|
601
877
|
analyser = null;
|
|
602
878
|
lipSyncInterval = null;
|
|
603
879
|
isResponseInProgress = false;
|
|
@@ -607,13 +883,33 @@ var OpenAIRealtimeClient = class {
|
|
|
607
883
|
assistantText = "";
|
|
608
884
|
nextSessionUpdateId = 1;
|
|
609
885
|
pendingSessionUpdate = null;
|
|
886
|
+
currentSessionConfig;
|
|
887
|
+
connectionLossNotified = false;
|
|
888
|
+
isExplicitDisconnect = false;
|
|
889
|
+
isRecovering = false;
|
|
890
|
+
isCleaningUp = false;
|
|
891
|
+
teardownTransportLifecycle;
|
|
892
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
893
|
+
this.emitConnectionLost("ice-disconnected");
|
|
894
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
610
895
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
611
896
|
async connect(config) {
|
|
612
897
|
try {
|
|
898
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
899
|
+
throw new Error("Realtime session disconnect requested");
|
|
900
|
+
}
|
|
613
901
|
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
902
|
+
this.isExplicitDisconnect = false;
|
|
903
|
+
this.isRecovering = false;
|
|
904
|
+
this.connectionLossNotified = false;
|
|
905
|
+
this.currentSessionConfig = config;
|
|
614
906
|
this.pc = new RTCPeerConnection();
|
|
615
907
|
this.audioElement = document.createElement("audio");
|
|
616
908
|
this.audioElement.autoplay = true;
|
|
909
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
910
|
+
this.prepareAudioAnalysis();
|
|
911
|
+
this.bindConnectionEvents();
|
|
912
|
+
this.bindTransportLifecycleEvents();
|
|
617
913
|
this.pc.ontrack = (event) => {
|
|
618
914
|
if (this.audioElement) {
|
|
619
915
|
this.audioElement.srcObject = event.streams[0];
|
|
@@ -621,11 +917,9 @@ var OpenAIRealtimeClient = class {
|
|
|
621
917
|
this.setupAudioAnalysis(event.streams[0]);
|
|
622
918
|
};
|
|
623
919
|
try {
|
|
624
|
-
this.mediaStream = await
|
|
625
|
-
audio: true
|
|
626
|
-
});
|
|
920
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
627
921
|
const audioTrack = this.mediaStream.getTracks()[0];
|
|
628
|
-
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
922
|
+
this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
|
|
629
923
|
} catch {
|
|
630
924
|
throw new Error("Microphone access required for Realtime API");
|
|
631
925
|
}
|
|
@@ -664,6 +958,8 @@ var OpenAIRealtimeClient = class {
|
|
|
664
958
|
}
|
|
665
959
|
async disconnect() {
|
|
666
960
|
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
961
|
+
this.isExplicitDisconnect = true;
|
|
962
|
+
this.isRecovering = false;
|
|
667
963
|
this.cleanup();
|
|
668
964
|
}
|
|
669
965
|
async updateSession(config) {
|
|
@@ -699,8 +995,27 @@ var OpenAIRealtimeClient = class {
|
|
|
699
995
|
timeoutId
|
|
700
996
|
};
|
|
701
997
|
});
|
|
998
|
+
this.currentSessionConfig = config;
|
|
999
|
+
}
|
|
1000
|
+
async recover(config) {
|
|
1001
|
+
this.isRecovering = true;
|
|
1002
|
+
this.connectionLossNotified = true;
|
|
1003
|
+
this.currentSessionConfig = config;
|
|
1004
|
+
try {
|
|
1005
|
+
if (await this.tryIceRestartRecovery(config)) {
|
|
1006
|
+
this.connectionLossNotified = false;
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
await this.rebuildConnection(config);
|
|
1010
|
+
this.connectionLossNotified = false;
|
|
1011
|
+
} finally {
|
|
1012
|
+
this.isRecovering = false;
|
|
1013
|
+
}
|
|
702
1014
|
}
|
|
703
1015
|
async sendText(text) {
|
|
1016
|
+
if (this.isRecovering) {
|
|
1017
|
+
throw new Error("Realtime transport reconnecting");
|
|
1018
|
+
}
|
|
704
1019
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
705
1020
|
throw new Error("DataChannel not ready");
|
|
706
1021
|
}
|
|
@@ -727,11 +1042,17 @@ var OpenAIRealtimeClient = class {
|
|
|
727
1042
|
this.beginResponseRequest();
|
|
728
1043
|
}
|
|
729
1044
|
async sendAudio(_audio) {
|
|
1045
|
+
if (this.isRecovering) {
|
|
1046
|
+
throw new Error("Realtime transport reconnecting");
|
|
1047
|
+
}
|
|
730
1048
|
console.warn(
|
|
731
1049
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
732
1050
|
);
|
|
733
1051
|
}
|
|
734
1052
|
async sendToolResult(callId, output) {
|
|
1053
|
+
if (this.isRecovering) {
|
|
1054
|
+
throw new Error("Realtime transport reconnecting");
|
|
1055
|
+
}
|
|
735
1056
|
const dc = this.requireOpenDataChannel();
|
|
736
1057
|
dc.send(
|
|
737
1058
|
JSON.stringify({
|
|
@@ -747,6 +1068,9 @@ var OpenAIRealtimeClient = class {
|
|
|
747
1068
|
this.beginResponseRequest();
|
|
748
1069
|
}
|
|
749
1070
|
async interrupt() {
|
|
1071
|
+
if (this.isRecovering) {
|
|
1072
|
+
throw new Error("Realtime transport reconnecting");
|
|
1073
|
+
}
|
|
750
1074
|
const dc = this.requireOpenDataChannel();
|
|
751
1075
|
if (!this.isResponseInProgress) {
|
|
752
1076
|
return;
|
|
@@ -943,18 +1267,133 @@ var OpenAIRealtimeClient = class {
|
|
|
943
1267
|
}
|
|
944
1268
|
return bootstrap;
|
|
945
1269
|
}
|
|
1270
|
+
bindConnectionEvents() {
|
|
1271
|
+
if (!this.pc) {
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
1275
|
+
const iceState = this.pc?.iceConnectionState;
|
|
1276
|
+
if (iceState === "failed") {
|
|
1277
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1278
|
+
this.emitConnectionLost("ice-failed");
|
|
1279
|
+
} else if (iceState === "disconnected") {
|
|
1280
|
+
this.iceDisconnectDebouncer.schedule();
|
|
1281
|
+
} else {
|
|
1282
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1283
|
+
}
|
|
1284
|
+
});
|
|
1285
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
1286
|
+
const connectionState = this.pc?.connectionState;
|
|
1287
|
+
if (connectionState === "failed" || connectionState === "closed") {
|
|
1288
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1289
|
+
this.emitConnectionLost("connection-failed");
|
|
1290
|
+
}
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
bindTransportLifecycleEvents() {
|
|
1294
|
+
if (this.teardownTransportLifecycle) {
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
1298
|
+
onHidden: this.handleHidden,
|
|
1299
|
+
onOnline: this.handleOnline,
|
|
1300
|
+
onPageHide: this.handlePageHide,
|
|
1301
|
+
onPageShow: this.handlePageShow,
|
|
1302
|
+
onVisible: this.handleVisible,
|
|
1303
|
+
onDeviceChange: this.handleDeviceChange
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
unbindTransportLifecycleEvents() {
|
|
1307
|
+
this.teardownTransportLifecycle?.();
|
|
1308
|
+
this.teardownTransportLifecycle = void 0;
|
|
1309
|
+
}
|
|
1310
|
+
handleOnline = () => {
|
|
1311
|
+
this.emitConnectionLost("online");
|
|
1312
|
+
};
|
|
1313
|
+
handleHidden = () => {
|
|
1314
|
+
this.pauseAudioAnalysis();
|
|
1315
|
+
};
|
|
1316
|
+
handleVisible = () => {
|
|
1317
|
+
this.resumeAudioAnalysis();
|
|
1318
|
+
this.emitConnectionLost("visibility");
|
|
1319
|
+
};
|
|
1320
|
+
handlePageHide = () => {
|
|
1321
|
+
this.pauseAudioAnalysis();
|
|
1322
|
+
};
|
|
1323
|
+
handlePageShow = (event) => {
|
|
1324
|
+
this.resumeAudioAnalysis();
|
|
1325
|
+
if (event.persisted) {
|
|
1326
|
+
this.emitConnectionLost("pageshow");
|
|
1327
|
+
}
|
|
1328
|
+
};
|
|
1329
|
+
handleDeviceChange = () => {
|
|
1330
|
+
void this.refreshMicrophoneTrack();
|
|
1331
|
+
};
|
|
1332
|
+
async refreshMicrophoneTrack() {
|
|
1333
|
+
if (!this.pc || this.isExplicitDisconnect) {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
try {
|
|
1337
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
1338
|
+
audioSender: this.audioSender,
|
|
1339
|
+
mediaStream: this.mediaStream,
|
|
1340
|
+
peerConnection: this.pc
|
|
1341
|
+
});
|
|
1342
|
+
this.audioSender = audioSender;
|
|
1343
|
+
this.mediaStream = mediaStream;
|
|
1344
|
+
} catch (error) {
|
|
1345
|
+
this.emitEvent({
|
|
1346
|
+
type: "error",
|
|
1347
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
emitConnectionLost(cause, error) {
|
|
1352
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
|
|
1353
|
+
return;
|
|
1354
|
+
}
|
|
1355
|
+
this.connectionLossNotified = true;
|
|
1356
|
+
this.isResponseInProgress = false;
|
|
1357
|
+
this.cancelInFlight = false;
|
|
1358
|
+
this.resetResponseTracking();
|
|
1359
|
+
this.stopLipSyncAnalysis();
|
|
1360
|
+
this.emitEvent({
|
|
1361
|
+
type: "connection.lost",
|
|
1362
|
+
cause,
|
|
1363
|
+
error
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
needsRecovery(cause) {
|
|
1367
|
+
if (!this.pc || !this.dc) {
|
|
1368
|
+
return false;
|
|
1369
|
+
}
|
|
1370
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
1371
|
+
return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
|
|
1372
|
+
}
|
|
1373
|
+
return true;
|
|
1374
|
+
}
|
|
1375
|
+
prepareAudioAnalysis() {
|
|
1376
|
+
if (this.audioContext) {
|
|
1377
|
+
return;
|
|
1378
|
+
}
|
|
1379
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
1380
|
+
if (!audioContextConstructor) {
|
|
1381
|
+
return;
|
|
1382
|
+
}
|
|
1383
|
+
this.audioContext = new audioContextConstructor();
|
|
1384
|
+
}
|
|
946
1385
|
setupAudioAnalysis(stream) {
|
|
947
1386
|
try {
|
|
948
|
-
|
|
949
|
-
if (!
|
|
950
|
-
|
|
1387
|
+
this.prepareAudioAnalysis();
|
|
1388
|
+
if (!this.audioContext) {
|
|
1389
|
+
return;
|
|
951
1390
|
}
|
|
952
|
-
this.
|
|
953
|
-
|
|
1391
|
+
this.audioSource?.disconnect();
|
|
1392
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
954
1393
|
this.analyser = this.audioContext.createAnalyser();
|
|
955
1394
|
this.analyser.fftSize = 256;
|
|
956
1395
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
957
|
-
|
|
1396
|
+
this.audioSource.connect(this.analyser);
|
|
958
1397
|
this.startLipSyncAnalysis();
|
|
959
1398
|
} catch (error) {
|
|
960
1399
|
console.error("Failed to setup audio analysis:", error);
|
|
@@ -990,7 +1429,19 @@ var OpenAIRealtimeClient = class {
|
|
|
990
1429
|
}
|
|
991
1430
|
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
992
1431
|
}
|
|
1432
|
+
pauseAudioAnalysis() {
|
|
1433
|
+
this.stopLipSyncAnalysis();
|
|
1434
|
+
}
|
|
1435
|
+
resumeAudioAnalysis() {
|
|
1436
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
this.startLipSyncAnalysis();
|
|
1440
|
+
}
|
|
993
1441
|
cleanup() {
|
|
1442
|
+
this.isCleaningUp = true;
|
|
1443
|
+
this.unbindTransportLifecycleEvents();
|
|
1444
|
+
this.iceDisconnectDebouncer.cancel();
|
|
994
1445
|
this.stopLipSyncAnalysis();
|
|
995
1446
|
this.rejectPendingSessionUpdate(
|
|
996
1447
|
new Error("Realtime session ended before session update completed")
|
|
@@ -1015,9 +1466,13 @@ var OpenAIRealtimeClient = class {
|
|
|
1015
1466
|
this.audioContext.close();
|
|
1016
1467
|
this.audioContext = null;
|
|
1017
1468
|
}
|
|
1469
|
+
this.audioSource = null;
|
|
1470
|
+
this.audioSender = null;
|
|
1018
1471
|
this.analyser = null;
|
|
1019
1472
|
this.isResponseInProgress = false;
|
|
1020
1473
|
this.cancelInFlight = false;
|
|
1474
|
+
this.connectionLossNotified = false;
|
|
1475
|
+
this.isCleaningUp = false;
|
|
1021
1476
|
this.resetResponseTracking();
|
|
1022
1477
|
}
|
|
1023
1478
|
beginResponseRequest() {
|
|
@@ -1026,11 +1481,60 @@ var OpenAIRealtimeClient = class {
|
|
|
1026
1481
|
this.cancelInFlight = false;
|
|
1027
1482
|
}
|
|
1028
1483
|
requireOpenDataChannel() {
|
|
1484
|
+
if (this.isRecovering) {
|
|
1485
|
+
throw new Error("Realtime transport reconnecting");
|
|
1486
|
+
}
|
|
1029
1487
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
1030
1488
|
throw new Error("DataChannel not ready");
|
|
1031
1489
|
}
|
|
1032
1490
|
return this.dc;
|
|
1033
1491
|
}
|
|
1492
|
+
async tryIceRestartRecovery(config) {
|
|
1493
|
+
const pc = this.pc;
|
|
1494
|
+
if (!pc || this.shouldRebuildConnection()) {
|
|
1495
|
+
return false;
|
|
1496
|
+
}
|
|
1497
|
+
if (pc.iceConnectionState === "connected") {
|
|
1498
|
+
return true;
|
|
1499
|
+
}
|
|
1500
|
+
if (pc.iceConnectionState === "disconnected") {
|
|
1501
|
+
await delay(1500);
|
|
1502
|
+
if (pc.iceConnectionState !== "disconnected") {
|
|
1503
|
+
return true;
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
1507
|
+
return false;
|
|
1508
|
+
}
|
|
1509
|
+
if (typeof pc.restartIce === "function") {
|
|
1510
|
+
pc.restartIce();
|
|
1511
|
+
}
|
|
1512
|
+
const offer = await pc.createOffer({ iceRestart: true });
|
|
1513
|
+
await pc.setLocalDescription(offer);
|
|
1514
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
1515
|
+
transport: "webrtc",
|
|
1516
|
+
session: config ?? this.currentSessionConfig ?? {},
|
|
1517
|
+
sdpOffer: offer.sdp
|
|
1518
|
+
});
|
|
1519
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
1520
|
+
throw new Error(
|
|
1521
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1522
|
+
);
|
|
1523
|
+
}
|
|
1524
|
+
await pc.setRemoteDescription({
|
|
1525
|
+
type: "answer",
|
|
1526
|
+
sdp: bootstrap.answerSdp
|
|
1527
|
+
});
|
|
1528
|
+
return true;
|
|
1529
|
+
}
|
|
1530
|
+
shouldRebuildConnection() {
|
|
1531
|
+
return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
|
|
1532
|
+
}
|
|
1533
|
+
async rebuildConnection(config) {
|
|
1534
|
+
this.connectionLossNotified = true;
|
|
1535
|
+
this.cleanup();
|
|
1536
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
1537
|
+
}
|
|
1034
1538
|
resolvePendingSessionUpdate() {
|
|
1035
1539
|
if (!this.pendingSessionUpdate) {
|
|
1036
1540
|
return;
|
|
@@ -1105,6 +1609,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
1105
1609
|
}
|
|
1106
1610
|
return session;
|
|
1107
1611
|
}
|
|
1612
|
+
function delay(ms) {
|
|
1613
|
+
return new Promise((resolve) => {
|
|
1614
|
+
setTimeout(resolve, ms);
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1108
1617
|
|
|
1109
1618
|
// src/remote/index.ts
|
|
1110
1619
|
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|
|
@@ -1156,6 +1665,9 @@ var RemoteRealtimeClient = class {
|
|
|
1156
1665
|
async updateSession(config) {
|
|
1157
1666
|
await this.getActiveTransportClient().updateSession(config);
|
|
1158
1667
|
}
|
|
1668
|
+
async recover(config) {
|
|
1669
|
+
await this.getActiveTransportClient().recover(config);
|
|
1670
|
+
}
|
|
1159
1671
|
async disconnect() {
|
|
1160
1672
|
if (!this.transportClient) {
|
|
1161
1673
|
return;
|