@charivo/realtime 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-WLTIMNHQ.mjs → chunk-DHNRDHST.mjs} +248 -13
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/{chunk-FABRM56W.mjs → chunk-PTFZ7CML.mjs} +208 -12
  5. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  6. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  7. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +217 -3
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +214 -3
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +323 -10
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +283 -9
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +531 -19
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +6 -3
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/remote/index.ts
7
7
 
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
85
+
8
86
  // src/internal/shared.ts
9
87
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
88
  function isRecord(value) {
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
83
161
  }
84
162
  audioAnalysisStream = null;
85
163
  audioContext = null;
164
+ audioSource = null;
86
165
  analyser = null;
87
166
  lipSyncInterval = null;
88
167
  pendingAudioElementPoll = null;
89
168
  observedAudioElement = null;
90
169
  observedAudioElementListener = null;
170
+ async prepareAudioContext() {
171
+ if (this.audioContext) {
172
+ return;
173
+ }
174
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
175
+ if (!audioContextConstructor) {
176
+ throw new Error("AudioContext is not supported in this browser");
177
+ }
178
+ this.audioContext = new audioContextConstructor();
179
+ }
91
180
  observeAudioElement(audioElement) {
92
181
  this.detachObservedAudioElement();
93
182
  const tryAttachStream = () => {
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
109
198
  this.stopLipSyncAnalysis();
110
199
  this.audioAnalysisStream = stream;
111
200
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
201
+ if (!this.audioContext) {
202
+ void this.prepareAudioContext().catch((error) => {
203
+ this.options.onError?.(error);
204
+ });
115
205
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
206
+ if (!this.audioContext) {
207
+ return;
208
+ }
209
+ this.audioSource?.disconnect?.();
210
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
211
  this.analyser = this.audioContext.createAnalyser();
119
212
  this.analyser.fftSize = 256;
120
213
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
214
+ this.audioSource.connect(this.analyser);
122
215
  this.startLipSyncAnalysis();
123
216
  } catch (error) {
124
217
  this.options.onError?.(error);
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
127
220
  stopOutput() {
128
221
  this.stopLipSyncAnalysis();
129
222
  }
223
+ pause() {
224
+ this.stopLipSyncAnalysis();
225
+ }
226
+ resume() {
227
+ if (!this.analyser || this.lipSyncInterval) {
228
+ return;
229
+ }
230
+ this.startLipSyncAnalysis();
231
+ }
130
232
  cleanup() {
131
233
  this.stopLipSyncAnalysis();
132
234
  this.stopAudioElementPolling();
133
235
  this.detachObservedAudioElement();
236
+ this.audioSource?.disconnect?.();
237
+ this.audioSource = null;
134
238
  if (this.audioContext) {
135
239
  void this.audioContext.close();
136
240
  this.audioContext = null;
@@ -265,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
265
369
  session = null;
266
370
  transport = null;
267
371
  audioElement = null;
372
+ mediaStream = null;
373
+ peerConnection = null;
374
+ audioSender = null;
268
375
  connectionWasActive = false;
376
+ connectionLossNotified = false;
377
+ isExplicitDisconnect = false;
378
+ isRecovering = false;
379
+ isCleaningUp = false;
269
380
  assistant = { text: "", started: false };
270
381
  latestHistory = [];
382
+ currentSessionConfig;
383
+ teardownTransportLifecycle;
384
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
385
+ this.emitConnectionLost("ice-disconnected");
386
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
271
387
  eventCallbacks = /* @__PURE__ */ new Set();
272
388
  pendingToolCalls = /* @__PURE__ */ new Map();
273
389
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -280,13 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
280
396
  });
281
397
  async connect(config) {
282
398
  try {
399
+ if (this.isExplicitDisconnect && this.isRecovering) {
400
+ throw new Error("Realtime session disconnect requested");
401
+ }
283
402
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
403
+ this.isExplicitDisconnect = false;
404
+ this.isRecovering = false;
405
+ this.connectionLossNotified = false;
406
+ this.currentSessionConfig = config;
284
407
  this.audioElement = document.createElement("audio");
285
408
  this.audioElement.autoplay = true;
286
409
  this.audioElement.setAttribute("playsinline", "true");
410
+ this.mediaStream = await acquireMicrophoneStream();
411
+ this.bindTransportLifecycleEvents();
412
+ await this.lipSyncAnalyzer.prepareAudioContext();
287
413
  const agent = this.createAgent(config);
288
414
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
289
- this.createTransportOptions(this.audioElement)
415
+ this.createTransportOptions(this.audioElement, this.mediaStream)
290
416
  );
291
417
  this.session = new agentsRealtime.RealtimeSession(agent, {
292
418
  transport: this.transport,
@@ -318,6 +444,8 @@ var OpenAIRealtimeAgentsClient = class {
318
444
  }
319
445
  async disconnect() {
320
446
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
447
+ this.isExplicitDisconnect = true;
448
+ this.isRecovering = false;
321
449
  this.cleanup();
322
450
  }
323
451
  async updateSession(config) {
@@ -326,8 +454,27 @@ var OpenAIRealtimeAgentsClient = class {
326
454
  }
327
455
  this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
328
456
  await this.session.updateAgent(this.createAgent(config));
457
+ this.currentSessionConfig = config;
458
+ }
459
+ async recover(config) {
460
+ this.isRecovering = true;
461
+ this.connectionLossNotified = true;
462
+ this.currentSessionConfig = config;
463
+ try {
464
+ this.cleanupPendingToolCalls(
465
+ new Error("Realtime session interrupted during reconnect")
466
+ );
467
+ this.cleanup();
468
+ await this.connect(config ?? this.currentSessionConfig);
469
+ this.connectionLossNotified = false;
470
+ } finally {
471
+ this.isRecovering = false;
472
+ }
329
473
  }
330
474
  async sendText(text) {
475
+ if (this.isRecovering) {
476
+ throw new Error("Realtime transport reconnecting");
477
+ }
331
478
  if (!this.session) {
332
479
  throw new Error("Realtime session not active");
333
480
  }
@@ -335,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
335
482
  this.session.sendMessage(text);
336
483
  }
337
484
  async sendAudio(_audio) {
485
+ if (this.isRecovering) {
486
+ throw new Error("Realtime transport reconnecting");
487
+ }
338
488
  console.warn(
339
489
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
340
490
  );
341
491
  }
342
492
  async sendToolResult(callId, output) {
493
+ if (this.isRecovering) {
494
+ throw new Error("Realtime transport reconnecting");
495
+ }
343
496
  const pendingCall = this.pendingToolCalls.get(callId);
344
497
  if (!pendingCall) {
345
498
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -348,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
348
501
  pendingCall.resolve(output);
349
502
  }
350
503
  async interrupt() {
504
+ if (this.isRecovering) {
505
+ throw new Error("Realtime transport reconnecting");
506
+ }
351
507
  if (!this.session) {
352
508
  throw new Error("Realtime session not active");
353
509
  }
@@ -411,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
411
567
  if (status === "disconnected" && this.connectionWasActive) {
412
568
  this.cleanupPendingToolCalls();
413
569
  this.connectionWasActive = false;
414
- this.emitEvent({ type: "session.ended" });
570
+ this.emitConnectionLost("connection-failed");
415
571
  }
416
572
  });
417
573
  }
@@ -513,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
513
669
  });
514
670
  });
515
671
  }
516
- createTransportOptions(audioElement) {
672
+ createTransportOptions(audioElement, mediaStream) {
517
673
  return {
518
674
  audioElement,
675
+ mediaStream,
519
676
  changePeerConnection: async (peerConnection) => {
677
+ this.peerConnection = peerConnection;
678
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
679
+ this.bindPeerConnectionEvents(peerConnection);
520
680
  peerConnection.addEventListener("track", (event) => {
521
681
  const stream = event.streams[0];
522
682
  if (stream) {
@@ -527,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
527
687
  }
528
688
  };
529
689
  }
690
+ bindPeerConnectionEvents(peerConnection) {
691
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
692
+ const iceState = peerConnection.iceConnectionState;
693
+ if (iceState === "failed") {
694
+ this.iceDisconnectDebouncer.cancel();
695
+ this.emitConnectionLost("ice-failed");
696
+ } else if (iceState === "disconnected") {
697
+ this.iceDisconnectDebouncer.schedule();
698
+ } else {
699
+ this.iceDisconnectDebouncer.cancel();
700
+ }
701
+ });
702
+ peerConnection.addEventListener("connectionstatechange", () => {
703
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
704
+ this.iceDisconnectDebouncer.cancel();
705
+ this.emitConnectionLost("connection-failed");
706
+ }
707
+ });
708
+ }
709
+ bindTransportLifecycleEvents() {
710
+ if (this.teardownTransportLifecycle) {
711
+ return;
712
+ }
713
+ this.teardownTransportLifecycle = bindTransportLifecycle({
714
+ onHidden: this.handleHidden,
715
+ onOnline: this.handleOnline,
716
+ onPageHide: this.handlePageHide,
717
+ onPageShow: this.handlePageShow,
718
+ onVisible: this.handleVisible,
719
+ onDeviceChange: this.handleDeviceChange
720
+ });
721
+ }
722
+ unbindTransportLifecycleEvents() {
723
+ this.teardownTransportLifecycle?.();
724
+ this.teardownTransportLifecycle = void 0;
725
+ }
726
+ handleOnline = () => {
727
+ this.emitConnectionLost("online");
728
+ };
729
+ handleHidden = () => {
730
+ this.lipSyncAnalyzer.pause();
731
+ };
732
+ handleVisible = () => {
733
+ this.lipSyncAnalyzer.resume();
734
+ this.emitConnectionLost("visibility");
735
+ };
736
+ handlePageHide = () => {
737
+ this.lipSyncAnalyzer.pause();
738
+ };
739
+ handlePageShow = (event) => {
740
+ this.lipSyncAnalyzer.resume();
741
+ if (event.persisted) {
742
+ this.emitConnectionLost("pageshow");
743
+ }
744
+ };
745
+ handleDeviceChange = () => {
746
+ void this.refreshMicrophoneTrack();
747
+ };
748
+ async refreshMicrophoneTrack() {
749
+ if (!this.peerConnection || this.isExplicitDisconnect) {
750
+ return;
751
+ }
752
+ try {
753
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
754
+ audioSender: this.audioSender,
755
+ mediaStream: this.mediaStream,
756
+ peerConnection: this.peerConnection
757
+ });
758
+ this.audioSender = audioSender;
759
+ this.mediaStream = mediaStream;
760
+ } catch (error) {
761
+ this.emitEvent({
762
+ type: "error",
763
+ error: error instanceof Error ? error : new Error(String(error))
764
+ });
765
+ }
766
+ }
767
+ emitConnectionLost(cause, error) {
768
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
769
+ return;
770
+ }
771
+ this.connectionLossNotified = true;
772
+ this.cleanupPendingToolCalls(
773
+ new Error("Realtime session interrupted during reconnect")
774
+ );
775
+ this.resetAssistantTracking();
776
+ this.lipSyncAnalyzer.stopOutput();
777
+ this.emitEvent({
778
+ type: "connection.lost",
779
+ cause,
780
+ error
781
+ });
782
+ }
783
+ shouldRecover(cause) {
784
+ if (!this.peerConnection) {
785
+ return false;
786
+ }
787
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
788
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
789
+ }
790
+ return true;
791
+ }
530
792
  cleanup(error) {
793
+ this.connectionWasActive = false;
794
+ this.isCleaningUp = true;
795
+ this.iceDisconnectDebouncer.cancel();
796
+ this.unbindTransportLifecycleEvents();
531
797
  this.cleanupPendingToolCalls(error);
532
798
  this.lipSyncAnalyzer.cleanup();
533
799
  if (this.transport) {
@@ -542,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
542
808
  this.audioElement.srcObject = null;
543
809
  this.audioElement = null;
544
810
  }
811
+ if (this.mediaStream) {
812
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
813
+ this.mediaStream = null;
814
+ }
815
+ this.peerConnection = null;
816
+ this.audioSender = null;
545
817
  this.latestHistory = [];
818
+ this.connectionLossNotified = false;
819
+ this.isCleaningUp = false;
546
820
  this.resetAssistantTracking();
547
821
  }
548
822
  resetAssistantTracking() {
@@ -597,7 +871,9 @@ var OpenAIRealtimeClient = class {
597
871
  dc = null;
598
872
  audioElement = null;
599
873
  mediaStream = null;
874
+ audioSender = null;
600
875
  audioContext = null;
876
+ audioSource = null;
601
877
  analyser = null;
602
878
  lipSyncInterval = null;
603
879
  isResponseInProgress = false;
@@ -607,13 +883,33 @@ var OpenAIRealtimeClient = class {
607
883
  assistantText = "";
608
884
  nextSessionUpdateId = 1;
609
885
  pendingSessionUpdate = null;
886
+ currentSessionConfig;
887
+ connectionLossNotified = false;
888
+ isExplicitDisconnect = false;
889
+ isRecovering = false;
890
+ isCleaningUp = false;
891
+ teardownTransportLifecycle;
892
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
893
+ this.emitConnectionLost("ice-disconnected");
894
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
610
895
  eventCallbacks = /* @__PURE__ */ new Set();
611
896
  async connect(config) {
612
897
  try {
898
+ if (this.isExplicitDisconnect && this.isRecovering) {
899
+ throw new Error("Realtime session disconnect requested");
900
+ }
613
901
  this.log("Starting OpenAI Realtime WebRTC connection");
902
+ this.isExplicitDisconnect = false;
903
+ this.isRecovering = false;
904
+ this.connectionLossNotified = false;
905
+ this.currentSessionConfig = config;
614
906
  this.pc = new RTCPeerConnection();
615
907
  this.audioElement = document.createElement("audio");
616
908
  this.audioElement.autoplay = true;
909
+ this.audioElement.setAttribute("playsinline", "true");
910
+ this.prepareAudioAnalysis();
911
+ this.bindConnectionEvents();
912
+ this.bindTransportLifecycleEvents();
617
913
  this.pc.ontrack = (event) => {
618
914
  if (this.audioElement) {
619
915
  this.audioElement.srcObject = event.streams[0];
@@ -621,11 +917,9 @@ var OpenAIRealtimeClient = class {
621
917
  this.setupAudioAnalysis(event.streams[0]);
622
918
  };
623
919
  try {
624
- this.mediaStream = await navigator.mediaDevices.getUserMedia({
625
- audio: true
626
- });
920
+ this.mediaStream = await acquireMicrophoneStream();
627
921
  const audioTrack = this.mediaStream.getTracks()[0];
628
- this.pc.addTrack(audioTrack, this.mediaStream);
922
+ this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
629
923
  } catch {
630
924
  throw new Error("Microphone access required for Realtime API");
631
925
  }
@@ -664,6 +958,8 @@ var OpenAIRealtimeClient = class {
664
958
  }
665
959
  async disconnect() {
666
960
  this.log("Disconnecting OpenAI Realtime WebRTC");
961
+ this.isExplicitDisconnect = true;
962
+ this.isRecovering = false;
667
963
  this.cleanup();
668
964
  }
669
965
  async updateSession(config) {
@@ -699,8 +995,27 @@ var OpenAIRealtimeClient = class {
699
995
  timeoutId
700
996
  };
701
997
  });
998
+ this.currentSessionConfig = config;
999
+ }
1000
+ async recover(config) {
1001
+ this.isRecovering = true;
1002
+ this.connectionLossNotified = true;
1003
+ this.currentSessionConfig = config;
1004
+ try {
1005
+ if (await this.tryIceRestartRecovery(config)) {
1006
+ this.connectionLossNotified = false;
1007
+ return;
1008
+ }
1009
+ await this.rebuildConnection(config);
1010
+ this.connectionLossNotified = false;
1011
+ } finally {
1012
+ this.isRecovering = false;
1013
+ }
702
1014
  }
703
1015
  async sendText(text) {
1016
+ if (this.isRecovering) {
1017
+ throw new Error("Realtime transport reconnecting");
1018
+ }
704
1019
  if (!this.dc || this.dc.readyState !== "open") {
705
1020
  throw new Error("DataChannel not ready");
706
1021
  }
@@ -727,11 +1042,17 @@ var OpenAIRealtimeClient = class {
727
1042
  this.beginResponseRequest();
728
1043
  }
729
1044
  async sendAudio(_audio) {
1045
+ if (this.isRecovering) {
1046
+ throw new Error("Realtime transport reconnecting");
1047
+ }
730
1048
  console.warn(
731
1049
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
732
1050
  );
733
1051
  }
734
1052
  async sendToolResult(callId, output) {
1053
+ if (this.isRecovering) {
1054
+ throw new Error("Realtime transport reconnecting");
1055
+ }
735
1056
  const dc = this.requireOpenDataChannel();
736
1057
  dc.send(
737
1058
  JSON.stringify({
@@ -747,6 +1068,9 @@ var OpenAIRealtimeClient = class {
747
1068
  this.beginResponseRequest();
748
1069
  }
749
1070
  async interrupt() {
1071
+ if (this.isRecovering) {
1072
+ throw new Error("Realtime transport reconnecting");
1073
+ }
750
1074
  const dc = this.requireOpenDataChannel();
751
1075
  if (!this.isResponseInProgress) {
752
1076
  return;
@@ -943,18 +1267,133 @@ var OpenAIRealtimeClient = class {
943
1267
  }
944
1268
  return bootstrap;
945
1269
  }
1270
+ bindConnectionEvents() {
1271
+ if (!this.pc) {
1272
+ return;
1273
+ }
1274
+ this.pc.addEventListener("iceconnectionstatechange", () => {
1275
+ const iceState = this.pc?.iceConnectionState;
1276
+ if (iceState === "failed") {
1277
+ this.iceDisconnectDebouncer.cancel();
1278
+ this.emitConnectionLost("ice-failed");
1279
+ } else if (iceState === "disconnected") {
1280
+ this.iceDisconnectDebouncer.schedule();
1281
+ } else {
1282
+ this.iceDisconnectDebouncer.cancel();
1283
+ }
1284
+ });
1285
+ this.pc.addEventListener("connectionstatechange", () => {
1286
+ const connectionState = this.pc?.connectionState;
1287
+ if (connectionState === "failed" || connectionState === "closed") {
1288
+ this.iceDisconnectDebouncer.cancel();
1289
+ this.emitConnectionLost("connection-failed");
1290
+ }
1291
+ });
1292
+ }
1293
+ bindTransportLifecycleEvents() {
1294
+ if (this.teardownTransportLifecycle) {
1295
+ return;
1296
+ }
1297
+ this.teardownTransportLifecycle = bindTransportLifecycle({
1298
+ onHidden: this.handleHidden,
1299
+ onOnline: this.handleOnline,
1300
+ onPageHide: this.handlePageHide,
1301
+ onPageShow: this.handlePageShow,
1302
+ onVisible: this.handleVisible,
1303
+ onDeviceChange: this.handleDeviceChange
1304
+ });
1305
+ }
1306
+ unbindTransportLifecycleEvents() {
1307
+ this.teardownTransportLifecycle?.();
1308
+ this.teardownTransportLifecycle = void 0;
1309
+ }
1310
+ handleOnline = () => {
1311
+ this.emitConnectionLost("online");
1312
+ };
1313
+ handleHidden = () => {
1314
+ this.pauseAudioAnalysis();
1315
+ };
1316
+ handleVisible = () => {
1317
+ this.resumeAudioAnalysis();
1318
+ this.emitConnectionLost("visibility");
1319
+ };
1320
+ handlePageHide = () => {
1321
+ this.pauseAudioAnalysis();
1322
+ };
1323
+ handlePageShow = (event) => {
1324
+ this.resumeAudioAnalysis();
1325
+ if (event.persisted) {
1326
+ this.emitConnectionLost("pageshow");
1327
+ }
1328
+ };
1329
+ handleDeviceChange = () => {
1330
+ void this.refreshMicrophoneTrack();
1331
+ };
1332
+ async refreshMicrophoneTrack() {
1333
+ if (!this.pc || this.isExplicitDisconnect) {
1334
+ return;
1335
+ }
1336
+ try {
1337
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
1338
+ audioSender: this.audioSender,
1339
+ mediaStream: this.mediaStream,
1340
+ peerConnection: this.pc
1341
+ });
1342
+ this.audioSender = audioSender;
1343
+ this.mediaStream = mediaStream;
1344
+ } catch (error) {
1345
+ this.emitEvent({
1346
+ type: "error",
1347
+ error: error instanceof Error ? error : new Error(String(error))
1348
+ });
1349
+ }
1350
+ }
1351
+ emitConnectionLost(cause, error) {
1352
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
1353
+ return;
1354
+ }
1355
+ this.connectionLossNotified = true;
1356
+ this.isResponseInProgress = false;
1357
+ this.cancelInFlight = false;
1358
+ this.resetResponseTracking();
1359
+ this.stopLipSyncAnalysis();
1360
+ this.emitEvent({
1361
+ type: "connection.lost",
1362
+ cause,
1363
+ error
1364
+ });
1365
+ }
1366
+ needsRecovery(cause) {
1367
+ if (!this.pc || !this.dc) {
1368
+ return false;
1369
+ }
1370
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
1371
+ return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
1372
+ }
1373
+ return true;
1374
+ }
1375
+ prepareAudioAnalysis() {
1376
+ if (this.audioContext) {
1377
+ return;
1378
+ }
1379
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
1380
+ if (!audioContextConstructor) {
1381
+ return;
1382
+ }
1383
+ this.audioContext = new audioContextConstructor();
1384
+ }
946
1385
  setupAudioAnalysis(stream) {
947
1386
  try {
948
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
949
- if (!audioContextConstructor) {
950
- throw new Error("AudioContext is not supported in this browser");
1387
+ this.prepareAudioAnalysis();
1388
+ if (!this.audioContext) {
1389
+ return;
951
1390
  }
952
- this.audioContext = new audioContextConstructor();
953
- const source = this.audioContext.createMediaStreamSource(stream);
1391
+ this.audioSource?.disconnect();
1392
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
954
1393
  this.analyser = this.audioContext.createAnalyser();
955
1394
  this.analyser.fftSize = 256;
956
1395
  this.analyser.smoothingTimeConstant = 0.8;
957
- source.connect(this.analyser);
1396
+ this.audioSource.connect(this.analyser);
958
1397
  this.startLipSyncAnalysis();
959
1398
  } catch (error) {
960
1399
  console.error("Failed to setup audio analysis:", error);
@@ -990,7 +1429,19 @@ var OpenAIRealtimeClient = class {
990
1429
  }
991
1430
  this.emitEvent({ type: "audio.lipsync", rms: 0 });
992
1431
  }
1432
+ pauseAudioAnalysis() {
1433
+ this.stopLipSyncAnalysis();
1434
+ }
1435
+ resumeAudioAnalysis() {
1436
+ if (!this.analyser || this.lipSyncInterval) {
1437
+ return;
1438
+ }
1439
+ this.startLipSyncAnalysis();
1440
+ }
993
1441
  cleanup() {
1442
+ this.isCleaningUp = true;
1443
+ this.unbindTransportLifecycleEvents();
1444
+ this.iceDisconnectDebouncer.cancel();
994
1445
  this.stopLipSyncAnalysis();
995
1446
  this.rejectPendingSessionUpdate(
996
1447
  new Error("Realtime session ended before session update completed")
@@ -1015,9 +1466,13 @@ var OpenAIRealtimeClient = class {
1015
1466
  this.audioContext.close();
1016
1467
  this.audioContext = null;
1017
1468
  }
1469
+ this.audioSource = null;
1470
+ this.audioSender = null;
1018
1471
  this.analyser = null;
1019
1472
  this.isResponseInProgress = false;
1020
1473
  this.cancelInFlight = false;
1474
+ this.connectionLossNotified = false;
1475
+ this.isCleaningUp = false;
1021
1476
  this.resetResponseTracking();
1022
1477
  }
1023
1478
  beginResponseRequest() {
@@ -1026,11 +1481,60 @@ var OpenAIRealtimeClient = class {
1026
1481
  this.cancelInFlight = false;
1027
1482
  }
1028
1483
  requireOpenDataChannel() {
1484
+ if (this.isRecovering) {
1485
+ throw new Error("Realtime transport reconnecting");
1486
+ }
1029
1487
  if (!this.dc || this.dc.readyState !== "open") {
1030
1488
  throw new Error("DataChannel not ready");
1031
1489
  }
1032
1490
  return this.dc;
1033
1491
  }
1492
+ async tryIceRestartRecovery(config) {
1493
+ const pc = this.pc;
1494
+ if (!pc || this.shouldRebuildConnection()) {
1495
+ return false;
1496
+ }
1497
+ if (pc.iceConnectionState === "connected") {
1498
+ return true;
1499
+ }
1500
+ if (pc.iceConnectionState === "disconnected") {
1501
+ await delay(1500);
1502
+ if (pc.iceConnectionState !== "disconnected") {
1503
+ return true;
1504
+ }
1505
+ }
1506
+ if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
1507
+ return false;
1508
+ }
1509
+ if (typeof pc.restartIce === "function") {
1510
+ pc.restartIce();
1511
+ }
1512
+ const offer = await pc.createOffer({ iceRestart: true });
1513
+ await pc.setLocalDescription(offer);
1514
+ const bootstrap = await this.getSessionBootstrap({
1515
+ transport: "webrtc",
1516
+ session: config ?? this.currentSessionConfig ?? {},
1517
+ sdpOffer: offer.sdp
1518
+ });
1519
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1520
+ throw new Error(
1521
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1522
+ );
1523
+ }
1524
+ await pc.setRemoteDescription({
1525
+ type: "answer",
1526
+ sdp: bootstrap.answerSdp
1527
+ });
1528
+ return true;
1529
+ }
1530
+ shouldRebuildConnection() {
1531
+ return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
1532
+ }
1533
+ async rebuildConnection(config) {
1534
+ this.connectionLossNotified = true;
1535
+ this.cleanup();
1536
+ await this.connect(config ?? this.currentSessionConfig);
1537
+ }
1034
1538
  resolvePendingSessionUpdate() {
1035
1539
  if (!this.pendingSessionUpdate) {
1036
1540
  return;
@@ -1105,6 +1609,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
1105
1609
  }
1106
1610
  return session;
1107
1611
  }
1612
+ function delay(ms) {
1613
+ return new Promise((resolve) => {
1614
+ setTimeout(resolve, ms);
1615
+ });
1616
+ }
1108
1617
 
1109
1618
  // src/remote/index.ts
1110
1619
  var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
@@ -1156,6 +1665,9 @@ var RemoteRealtimeClient = class {
1156
1665
  async updateSession(config) {
1157
1666
  await this.getActiveTransportClient().updateSession(config);
1158
1667
  }
1668
+ async recover(config) {
1669
+ await this.getActiveTransportClient().recover(config);
1670
+ }
1159
1671
  async disconnect() {
1160
1672
  if (!this.transportClient) {
1161
1673
  return;