@charivo/realtime 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-FABRM56W.mjs → chunk-3O3COMUY.mjs} +216 -17
  3. package/dist/chunk-3O3COMUY.mjs.map +1 -0
  4. package/dist/{chunk-WLTIMNHQ.mjs → chunk-J5L3BPMG.mjs} +248 -13
  5. package/dist/chunk-J5L3BPMG.mjs.map +1 -0
  6. package/dist/chunk-ZJZO4QYA.mjs +123 -0
  7. package/dist/chunk-ZJZO4QYA.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +240 -21
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +237 -21
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +326 -13
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +293 -17
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +572 -54
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +38 -31
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,7 +5,83 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/remote/index.ts
7
7
 
8
- // src/internal/shared.ts
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
9
85
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
86
  function isRecord(value) {
11
87
  return typeof value === "object" && value !== null;
@@ -32,9 +108,11 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
32
108
  });
33
109
  } catch (error) {
34
110
  if (isAbortError(error)) {
35
- throw new Error(timeoutMessage);
111
+ throw new core.CharivoTimeoutError(timeoutMessage, { cause: error });
36
112
  }
37
- throw error;
113
+ throw new core.CharivoTransportError("Realtime request failed", {
114
+ cause: error instanceof Error ? error : void 0
115
+ });
38
116
  } finally {
39
117
  clearTimeout(timeoutId);
40
118
  }
@@ -42,15 +120,13 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
42
120
  function isAbortError(error) {
43
121
  return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
44
122
  }
45
-
46
- // src/openai-agents/bootstrap.ts
47
123
  async function getOpenAIRealtimeAgentsBootstrap(options, request) {
48
124
  if (options.sessionBootstrap) {
49
125
  return options.sessionBootstrap(request);
50
126
  }
51
127
  const apiEndpoint = options.apiEndpoint;
52
128
  if (!apiEndpoint) {
53
- throw new Error(
129
+ throw new core.CharivoStateError(
54
130
  "OpenAI agents realtime client requires apiEndpoint or sessionBootstrap"
55
131
  );
56
132
  }
@@ -67,11 +143,15 @@ async function getOpenAIRealtimeAgentsBootstrap(options, request) {
67
143
  );
68
144
  if (!response.ok) {
69
145
  const errorText = await response.text();
70
- throw new Error(`Failed to create Realtime session: ${errorText}`);
146
+ throw new core.CharivoProviderError(
147
+ `Failed to create Realtime session: ${errorText}`
148
+ );
71
149
  }
72
150
  const bootstrap = await response.json();
73
151
  if (!isRealtimeSessionBootstrap(bootstrap)) {
74
- throw new Error("Invalid realtime session bootstrap response");
152
+ throw new core.CharivoProviderError(
153
+ "Invalid realtime session bootstrap response"
154
+ );
75
155
  }
76
156
  return bootstrap;
77
157
  }
@@ -83,11 +163,22 @@ var LipSyncAnalyzer = class {
83
163
  }
84
164
  audioAnalysisStream = null;
85
165
  audioContext = null;
166
+ audioSource = null;
86
167
  analyser = null;
87
168
  lipSyncInterval = null;
88
169
  pendingAudioElementPoll = null;
89
170
  observedAudioElement = null;
90
171
  observedAudioElementListener = null;
172
+ async prepareAudioContext() {
173
+ if (this.audioContext) {
174
+ return;
175
+ }
176
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
177
+ if (!audioContextConstructor) {
178
+ throw new Error("AudioContext is not supported in this browser");
179
+ }
180
+ this.audioContext = new audioContextConstructor();
181
+ }
91
182
  observeAudioElement(audioElement) {
92
183
  this.detachObservedAudioElement();
93
184
  const tryAttachStream = () => {
@@ -109,16 +200,20 @@ var LipSyncAnalyzer = class {
109
200
  this.stopLipSyncAnalysis();
110
201
  this.audioAnalysisStream = stream;
111
202
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
203
+ if (!this.audioContext) {
204
+ void this.prepareAudioContext().catch((error) => {
205
+ this.options.onError?.(error);
206
+ });
207
+ }
208
+ if (!this.audioContext) {
209
+ return;
115
210
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
211
+ this.audioSource?.disconnect?.();
212
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
213
  this.analyser = this.audioContext.createAnalyser();
119
214
  this.analyser.fftSize = 256;
120
215
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
216
+ this.audioSource.connect(this.analyser);
122
217
  this.startLipSyncAnalysis();
123
218
  } catch (error) {
124
219
  this.options.onError?.(error);
@@ -127,10 +222,21 @@ var LipSyncAnalyzer = class {
127
222
  stopOutput() {
128
223
  this.stopLipSyncAnalysis();
129
224
  }
225
+ pause() {
226
+ this.stopLipSyncAnalysis();
227
+ }
228
+ resume() {
229
+ if (!this.analyser || this.lipSyncInterval) {
230
+ return;
231
+ }
232
+ this.startLipSyncAnalysis();
233
+ }
130
234
  cleanup() {
131
235
  this.stopLipSyncAnalysis();
132
236
  this.stopAudioElementPolling();
133
237
  this.detachObservedAudioElement();
238
+ this.audioSource?.disconnect?.();
239
+ this.audioSource = null;
134
240
  if (this.audioContext) {
135
241
  void this.audioContext.close();
136
242
  this.audioContext = null;
@@ -265,9 +371,21 @@ var OpenAIRealtimeAgentsClient = class {
265
371
  session = null;
266
372
  transport = null;
267
373
  audioElement = null;
374
+ mediaStream = null;
375
+ peerConnection = null;
376
+ audioSender = null;
268
377
  connectionWasActive = false;
378
+ connectionLossNotified = false;
379
+ isExplicitDisconnect = false;
380
+ isRecovering = false;
381
+ isCleaningUp = false;
269
382
  assistant = { text: "", started: false };
270
383
  latestHistory = [];
384
+ currentSessionConfig;
385
+ teardownTransportLifecycle;
386
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
387
+ this.emitConnectionLost("ice-disconnected");
388
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
271
389
  eventCallbacks = /* @__PURE__ */ new Set();
272
390
  pendingToolCalls = /* @__PURE__ */ new Map();
273
391
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -280,13 +398,23 @@ var OpenAIRealtimeAgentsClient = class {
280
398
  });
281
399
  async connect(config) {
282
400
  try {
401
+ if (this.isExplicitDisconnect && this.isRecovering) {
402
+ throw new Error("Realtime session disconnect requested");
403
+ }
283
404
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
405
+ this.isExplicitDisconnect = false;
406
+ this.isRecovering = false;
407
+ this.connectionLossNotified = false;
408
+ this.currentSessionConfig = config;
284
409
  this.audioElement = document.createElement("audio");
285
410
  this.audioElement.autoplay = true;
286
411
  this.audioElement.setAttribute("playsinline", "true");
412
+ this.mediaStream = await acquireMicrophoneStream();
413
+ this.bindTransportLifecycleEvents();
414
+ await this.lipSyncAnalyzer.prepareAudioContext();
287
415
  const agent = this.createAgent(config);
288
416
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
289
- this.createTransportOptions(this.audioElement)
417
+ this.createTransportOptions(this.audioElement, this.mediaStream)
290
418
  );
291
419
  this.session = new agentsRealtime.RealtimeSession(agent, {
292
420
  transport: this.transport,
@@ -318,6 +446,8 @@ var OpenAIRealtimeAgentsClient = class {
318
446
  }
319
447
  async disconnect() {
320
448
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
449
+ this.isExplicitDisconnect = true;
450
+ this.isRecovering = false;
321
451
  this.cleanup();
322
452
  }
323
453
  async updateSession(config) {
@@ -326,8 +456,27 @@ var OpenAIRealtimeAgentsClient = class {
326
456
  }
327
457
  this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
328
458
  await this.session.updateAgent(this.createAgent(config));
459
+ this.currentSessionConfig = config;
460
+ }
461
+ async recover(config) {
462
+ this.isRecovering = true;
463
+ this.connectionLossNotified = true;
464
+ this.currentSessionConfig = config;
465
+ try {
466
+ this.cleanupPendingToolCalls(
467
+ new Error("Realtime session interrupted during reconnect")
468
+ );
469
+ this.cleanup();
470
+ await this.connect(config ?? this.currentSessionConfig);
471
+ this.connectionLossNotified = false;
472
+ } finally {
473
+ this.isRecovering = false;
474
+ }
329
475
  }
330
476
  async sendText(text) {
477
+ if (this.isRecovering) {
478
+ throw new Error("Realtime transport reconnecting");
479
+ }
331
480
  if (!this.session) {
332
481
  throw new Error("Realtime session not active");
333
482
  }
@@ -335,11 +484,17 @@ var OpenAIRealtimeAgentsClient = class {
335
484
  this.session.sendMessage(text);
336
485
  }
337
486
  async sendAudio(_audio) {
487
+ if (this.isRecovering) {
488
+ throw new Error("Realtime transport reconnecting");
489
+ }
338
490
  console.warn(
339
491
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
340
492
  );
341
493
  }
342
494
  async sendToolResult(callId, output) {
495
+ if (this.isRecovering) {
496
+ throw new Error("Realtime transport reconnecting");
497
+ }
343
498
  const pendingCall = this.pendingToolCalls.get(callId);
344
499
  if (!pendingCall) {
345
500
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -348,6 +503,9 @@ var OpenAIRealtimeAgentsClient = class {
348
503
  pendingCall.resolve(output);
349
504
  }
350
505
  async interrupt() {
506
+ if (this.isRecovering) {
507
+ throw new Error("Realtime transport reconnecting");
508
+ }
351
509
  if (!this.session) {
352
510
  throw new Error("Realtime session not active");
353
511
  }
@@ -411,7 +569,7 @@ var OpenAIRealtimeAgentsClient = class {
411
569
  if (status === "disconnected" && this.connectionWasActive) {
412
570
  this.cleanupPendingToolCalls();
413
571
  this.connectionWasActive = false;
414
- this.emitEvent({ type: "session.ended" });
572
+ this.emitConnectionLost("connection-failed");
415
573
  }
416
574
  });
417
575
  }
@@ -513,10 +671,14 @@ var OpenAIRealtimeAgentsClient = class {
513
671
  });
514
672
  });
515
673
  }
516
- createTransportOptions(audioElement) {
674
+ createTransportOptions(audioElement, mediaStream) {
517
675
  return {
518
676
  audioElement,
677
+ mediaStream,
519
678
  changePeerConnection: async (peerConnection) => {
679
+ this.peerConnection = peerConnection;
680
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
681
+ this.bindPeerConnectionEvents(peerConnection);
520
682
  peerConnection.addEventListener("track", (event) => {
521
683
  const stream = event.streams[0];
522
684
  if (stream) {
@@ -527,7 +689,113 @@ var OpenAIRealtimeAgentsClient = class {
527
689
  }
528
690
  };
529
691
  }
692
+ bindPeerConnectionEvents(peerConnection) {
693
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
694
+ const iceState = peerConnection.iceConnectionState;
695
+ if (iceState === "failed") {
696
+ this.iceDisconnectDebouncer.cancel();
697
+ this.emitConnectionLost("ice-failed");
698
+ } else if (iceState === "disconnected") {
699
+ this.iceDisconnectDebouncer.schedule();
700
+ } else {
701
+ this.iceDisconnectDebouncer.cancel();
702
+ }
703
+ });
704
+ peerConnection.addEventListener("connectionstatechange", () => {
705
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
706
+ this.iceDisconnectDebouncer.cancel();
707
+ this.emitConnectionLost("connection-failed");
708
+ }
709
+ });
710
+ }
711
+ bindTransportLifecycleEvents() {
712
+ if (this.teardownTransportLifecycle) {
713
+ return;
714
+ }
715
+ this.teardownTransportLifecycle = bindTransportLifecycle({
716
+ onHidden: this.handleHidden,
717
+ onOnline: this.handleOnline,
718
+ onPageHide: this.handlePageHide,
719
+ onPageShow: this.handlePageShow,
720
+ onVisible: this.handleVisible,
721
+ onDeviceChange: this.handleDeviceChange
722
+ });
723
+ }
724
+ unbindTransportLifecycleEvents() {
725
+ this.teardownTransportLifecycle?.();
726
+ this.teardownTransportLifecycle = void 0;
727
+ }
728
+ handleOnline = () => {
729
+ this.emitConnectionLost("online");
730
+ };
731
+ handleHidden = () => {
732
+ this.lipSyncAnalyzer.pause();
733
+ };
734
+ handleVisible = () => {
735
+ this.lipSyncAnalyzer.resume();
736
+ this.emitConnectionLost("visibility");
737
+ };
738
+ handlePageHide = () => {
739
+ this.lipSyncAnalyzer.pause();
740
+ };
741
+ handlePageShow = (event) => {
742
+ this.lipSyncAnalyzer.resume();
743
+ if (event.persisted) {
744
+ this.emitConnectionLost("pageshow");
745
+ }
746
+ };
747
+ handleDeviceChange = () => {
748
+ void this.refreshMicrophoneTrack();
749
+ };
750
+ async refreshMicrophoneTrack() {
751
+ if (!this.peerConnection || this.isExplicitDisconnect) {
752
+ return;
753
+ }
754
+ try {
755
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
756
+ audioSender: this.audioSender,
757
+ mediaStream: this.mediaStream,
758
+ peerConnection: this.peerConnection
759
+ });
760
+ this.audioSender = audioSender;
761
+ this.mediaStream = mediaStream;
762
+ } catch (error) {
763
+ this.emitEvent({
764
+ type: "error",
765
+ error: error instanceof Error ? error : new Error(String(error))
766
+ });
767
+ }
768
+ }
769
+ emitConnectionLost(cause, error) {
770
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
771
+ return;
772
+ }
773
+ this.connectionLossNotified = true;
774
+ this.cleanupPendingToolCalls(
775
+ new Error("Realtime session interrupted during reconnect")
776
+ );
777
+ this.resetAssistantTracking();
778
+ this.lipSyncAnalyzer.stopOutput();
779
+ this.emitEvent({
780
+ type: "connection.lost",
781
+ cause,
782
+ error
783
+ });
784
+ }
785
+ shouldRecover(cause) {
786
+ if (!this.peerConnection) {
787
+ return false;
788
+ }
789
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
790
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
791
+ }
792
+ return true;
793
+ }
530
794
  cleanup(error) {
795
+ this.connectionWasActive = false;
796
+ this.isCleaningUp = true;
797
+ this.iceDisconnectDebouncer.cancel();
798
+ this.unbindTransportLifecycleEvents();
531
799
  this.cleanupPendingToolCalls(error);
532
800
  this.lipSyncAnalyzer.cleanup();
533
801
  if (this.transport) {
@@ -542,7 +810,15 @@ var OpenAIRealtimeAgentsClient = class {
542
810
  this.audioElement.srcObject = null;
543
811
  this.audioElement = null;
544
812
  }
813
+ if (this.mediaStream) {
814
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
815
+ this.mediaStream = null;
816
+ }
817
+ this.peerConnection = null;
818
+ this.audioSender = null;
545
819
  this.latestHistory = [];
820
+ this.connectionLossNotified = false;
821
+ this.isCleaningUp = false;
546
822
  this.resetAssistantTracking();
547
823
  }
548
824
  resetAssistantTracking() {
@@ -597,7 +873,9 @@ var OpenAIRealtimeClient = class {
597
873
  dc = null;
598
874
  audioElement = null;
599
875
  mediaStream = null;
876
+ audioSender = null;
600
877
  audioContext = null;
878
+ audioSource = null;
601
879
  analyser = null;
602
880
  lipSyncInterval = null;
603
881
  isResponseInProgress = false;
@@ -607,13 +885,33 @@ var OpenAIRealtimeClient = class {
607
885
  assistantText = "";
608
886
  nextSessionUpdateId = 1;
609
887
  pendingSessionUpdate = null;
888
+ currentSessionConfig;
889
+ connectionLossNotified = false;
890
+ isExplicitDisconnect = false;
891
+ isRecovering = false;
892
+ isCleaningUp = false;
893
+ teardownTransportLifecycle;
894
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
895
+ this.emitConnectionLost("ice-disconnected");
896
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
610
897
  eventCallbacks = /* @__PURE__ */ new Set();
611
898
  async connect(config) {
612
899
  try {
900
+ if (this.isExplicitDisconnect && this.isRecovering) {
901
+ throw new Error("Realtime session disconnect requested");
902
+ }
613
903
  this.log("Starting OpenAI Realtime WebRTC connection");
904
+ this.isExplicitDisconnect = false;
905
+ this.isRecovering = false;
906
+ this.connectionLossNotified = false;
907
+ this.currentSessionConfig = config;
614
908
  this.pc = new RTCPeerConnection();
615
909
  this.audioElement = document.createElement("audio");
616
910
  this.audioElement.autoplay = true;
911
+ this.audioElement.setAttribute("playsinline", "true");
912
+ this.prepareAudioAnalysis();
913
+ this.bindConnectionEvents();
914
+ this.bindTransportLifecycleEvents();
617
915
  this.pc.ontrack = (event) => {
618
916
  if (this.audioElement) {
619
917
  this.audioElement.srcObject = event.streams[0];
@@ -621,11 +919,9 @@ var OpenAIRealtimeClient = class {
621
919
  this.setupAudioAnalysis(event.streams[0]);
622
920
  };
623
921
  try {
624
- this.mediaStream = await navigator.mediaDevices.getUserMedia({
625
- audio: true
626
- });
922
+ this.mediaStream = await acquireMicrophoneStream();
627
923
  const audioTrack = this.mediaStream.getTracks()[0];
628
- this.pc.addTrack(audioTrack, this.mediaStream);
924
+ this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
629
925
  } catch {
630
926
  throw new Error("Microphone access required for Realtime API");
631
927
  }
@@ -664,6 +960,8 @@ var OpenAIRealtimeClient = class {
664
960
  }
665
961
  async disconnect() {
666
962
  this.log("Disconnecting OpenAI Realtime WebRTC");
963
+ this.isExplicitDisconnect = true;
964
+ this.isRecovering = false;
667
965
  this.cleanup();
668
966
  }
669
967
  async updateSession(config) {
@@ -699,8 +997,27 @@ var OpenAIRealtimeClient = class {
699
997
  timeoutId
700
998
  };
701
999
  });
1000
+ this.currentSessionConfig = config;
1001
+ }
1002
+ async recover(config) {
1003
+ this.isRecovering = true;
1004
+ this.connectionLossNotified = true;
1005
+ this.currentSessionConfig = config;
1006
+ try {
1007
+ if (await this.tryIceRestartRecovery(config)) {
1008
+ this.connectionLossNotified = false;
1009
+ return;
1010
+ }
1011
+ await this.rebuildConnection(config);
1012
+ this.connectionLossNotified = false;
1013
+ } finally {
1014
+ this.isRecovering = false;
1015
+ }
702
1016
  }
703
1017
  async sendText(text) {
1018
+ if (this.isRecovering) {
1019
+ throw new Error("Realtime transport reconnecting");
1020
+ }
704
1021
  if (!this.dc || this.dc.readyState !== "open") {
705
1022
  throw new Error("DataChannel not ready");
706
1023
  }
@@ -727,11 +1044,17 @@ var OpenAIRealtimeClient = class {
727
1044
  this.beginResponseRequest();
728
1045
  }
729
1046
  async sendAudio(_audio) {
1047
+ if (this.isRecovering) {
1048
+ throw new Error("Realtime transport reconnecting");
1049
+ }
730
1050
  console.warn(
731
1051
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
732
1052
  );
733
1053
  }
734
1054
  async sendToolResult(callId, output) {
1055
+ if (this.isRecovering) {
1056
+ throw new Error("Realtime transport reconnecting");
1057
+ }
735
1058
  const dc = this.requireOpenDataChannel();
736
1059
  dc.send(
737
1060
  JSON.stringify({
@@ -747,6 +1070,9 @@ var OpenAIRealtimeClient = class {
747
1070
  this.beginResponseRequest();
748
1071
  }
749
1072
  async interrupt() {
1073
+ if (this.isRecovering) {
1074
+ throw new Error("Realtime transport reconnecting");
1075
+ }
750
1076
  const dc = this.requireOpenDataChannel();
751
1077
  if (!this.isResponseInProgress) {
752
1078
  return;
@@ -943,18 +1269,133 @@ var OpenAIRealtimeClient = class {
943
1269
  }
944
1270
  return bootstrap;
945
1271
  }
1272
+ bindConnectionEvents() {
1273
+ if (!this.pc) {
1274
+ return;
1275
+ }
1276
+ this.pc.addEventListener("iceconnectionstatechange", () => {
1277
+ const iceState = this.pc?.iceConnectionState;
1278
+ if (iceState === "failed") {
1279
+ this.iceDisconnectDebouncer.cancel();
1280
+ this.emitConnectionLost("ice-failed");
1281
+ } else if (iceState === "disconnected") {
1282
+ this.iceDisconnectDebouncer.schedule();
1283
+ } else {
1284
+ this.iceDisconnectDebouncer.cancel();
1285
+ }
1286
+ });
1287
+ this.pc.addEventListener("connectionstatechange", () => {
1288
+ const connectionState = this.pc?.connectionState;
1289
+ if (connectionState === "failed" || connectionState === "closed") {
1290
+ this.iceDisconnectDebouncer.cancel();
1291
+ this.emitConnectionLost("connection-failed");
1292
+ }
1293
+ });
1294
+ }
1295
+ bindTransportLifecycleEvents() {
1296
+ if (this.teardownTransportLifecycle) {
1297
+ return;
1298
+ }
1299
+ this.teardownTransportLifecycle = bindTransportLifecycle({
1300
+ onHidden: this.handleHidden,
1301
+ onOnline: this.handleOnline,
1302
+ onPageHide: this.handlePageHide,
1303
+ onPageShow: this.handlePageShow,
1304
+ onVisible: this.handleVisible,
1305
+ onDeviceChange: this.handleDeviceChange
1306
+ });
1307
+ }
1308
+ unbindTransportLifecycleEvents() {
1309
+ this.teardownTransportLifecycle?.();
1310
+ this.teardownTransportLifecycle = void 0;
1311
+ }
1312
+ handleOnline = () => {
1313
+ this.emitConnectionLost("online");
1314
+ };
1315
+ handleHidden = () => {
1316
+ this.pauseAudioAnalysis();
1317
+ };
1318
+ handleVisible = () => {
1319
+ this.resumeAudioAnalysis();
1320
+ this.emitConnectionLost("visibility");
1321
+ };
1322
+ handlePageHide = () => {
1323
+ this.pauseAudioAnalysis();
1324
+ };
1325
+ handlePageShow = (event) => {
1326
+ this.resumeAudioAnalysis();
1327
+ if (event.persisted) {
1328
+ this.emitConnectionLost("pageshow");
1329
+ }
1330
+ };
1331
+ handleDeviceChange = () => {
1332
+ void this.refreshMicrophoneTrack();
1333
+ };
1334
+ async refreshMicrophoneTrack() {
1335
+ if (!this.pc || this.isExplicitDisconnect) {
1336
+ return;
1337
+ }
1338
+ try {
1339
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
1340
+ audioSender: this.audioSender,
1341
+ mediaStream: this.mediaStream,
1342
+ peerConnection: this.pc
1343
+ });
1344
+ this.audioSender = audioSender;
1345
+ this.mediaStream = mediaStream;
1346
+ } catch (error) {
1347
+ this.emitEvent({
1348
+ type: "error",
1349
+ error: error instanceof Error ? error : new Error(String(error))
1350
+ });
1351
+ }
1352
+ }
1353
+ emitConnectionLost(cause, error) {
1354
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
1355
+ return;
1356
+ }
1357
+ this.connectionLossNotified = true;
1358
+ this.isResponseInProgress = false;
1359
+ this.cancelInFlight = false;
1360
+ this.resetResponseTracking();
1361
+ this.stopLipSyncAnalysis();
1362
+ this.emitEvent({
1363
+ type: "connection.lost",
1364
+ cause,
1365
+ error
1366
+ });
1367
+ }
1368
+ needsRecovery(cause) {
1369
+ if (!this.pc || !this.dc) {
1370
+ return false;
1371
+ }
1372
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
1373
+ return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
1374
+ }
1375
+ return true;
1376
+ }
1377
+ prepareAudioAnalysis() {
1378
+ if (this.audioContext) {
1379
+ return;
1380
+ }
1381
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
1382
+ if (!audioContextConstructor) {
1383
+ return;
1384
+ }
1385
+ this.audioContext = new audioContextConstructor();
1386
+ }
946
1387
  setupAudioAnalysis(stream) {
947
1388
  try {
948
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
949
- if (!audioContextConstructor) {
950
- throw new Error("AudioContext is not supported in this browser");
1389
+ this.prepareAudioAnalysis();
1390
+ if (!this.audioContext) {
1391
+ return;
951
1392
  }
952
- this.audioContext = new audioContextConstructor();
953
- const source = this.audioContext.createMediaStreamSource(stream);
1393
+ this.audioSource?.disconnect();
1394
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
954
1395
  this.analyser = this.audioContext.createAnalyser();
955
1396
  this.analyser.fftSize = 256;
956
1397
  this.analyser.smoothingTimeConstant = 0.8;
957
- source.connect(this.analyser);
1398
+ this.audioSource.connect(this.analyser);
958
1399
  this.startLipSyncAnalysis();
959
1400
  } catch (error) {
960
1401
  console.error("Failed to setup audio analysis:", error);
@@ -990,7 +1431,19 @@ var OpenAIRealtimeClient = class {
990
1431
  }
991
1432
  this.emitEvent({ type: "audio.lipsync", rms: 0 });
992
1433
  }
1434
+ pauseAudioAnalysis() {
1435
+ this.stopLipSyncAnalysis();
1436
+ }
1437
+ resumeAudioAnalysis() {
1438
+ if (!this.analyser || this.lipSyncInterval) {
1439
+ return;
1440
+ }
1441
+ this.startLipSyncAnalysis();
1442
+ }
993
1443
  cleanup() {
1444
+ this.isCleaningUp = true;
1445
+ this.unbindTransportLifecycleEvents();
1446
+ this.iceDisconnectDebouncer.cancel();
994
1447
  this.stopLipSyncAnalysis();
995
1448
  this.rejectPendingSessionUpdate(
996
1449
  new Error("Realtime session ended before session update completed")
@@ -1015,9 +1468,13 @@ var OpenAIRealtimeClient = class {
1015
1468
  this.audioContext.close();
1016
1469
  this.audioContext = null;
1017
1470
  }
1471
+ this.audioSource = null;
1472
+ this.audioSender = null;
1018
1473
  this.analyser = null;
1019
1474
  this.isResponseInProgress = false;
1020
1475
  this.cancelInFlight = false;
1476
+ this.connectionLossNotified = false;
1477
+ this.isCleaningUp = false;
1021
1478
  this.resetResponseTracking();
1022
1479
  }
1023
1480
  beginResponseRequest() {
@@ -1026,11 +1483,60 @@ var OpenAIRealtimeClient = class {
1026
1483
  this.cancelInFlight = false;
1027
1484
  }
1028
1485
  requireOpenDataChannel() {
1486
+ if (this.isRecovering) {
1487
+ throw new Error("Realtime transport reconnecting");
1488
+ }
1029
1489
  if (!this.dc || this.dc.readyState !== "open") {
1030
1490
  throw new Error("DataChannel not ready");
1031
1491
  }
1032
1492
  return this.dc;
1033
1493
  }
1494
+ async tryIceRestartRecovery(config) {
1495
+ const pc = this.pc;
1496
+ if (!pc || this.shouldRebuildConnection()) {
1497
+ return false;
1498
+ }
1499
+ if (pc.iceConnectionState === "connected") {
1500
+ return true;
1501
+ }
1502
+ if (pc.iceConnectionState === "disconnected") {
1503
+ await delay(1500);
1504
+ if (pc.iceConnectionState !== "disconnected") {
1505
+ return true;
1506
+ }
1507
+ }
1508
+ if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
1509
+ return false;
1510
+ }
1511
+ if (typeof pc.restartIce === "function") {
1512
+ pc.restartIce();
1513
+ }
1514
+ const offer = await pc.createOffer({ iceRestart: true });
1515
+ await pc.setLocalDescription(offer);
1516
+ const bootstrap = await this.getSessionBootstrap({
1517
+ transport: "webrtc",
1518
+ session: config ?? this.currentSessionConfig ?? {},
1519
+ sdpOffer: offer.sdp
1520
+ });
1521
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1522
+ throw new Error(
1523
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1524
+ );
1525
+ }
1526
+ await pc.setRemoteDescription({
1527
+ type: "answer",
1528
+ sdp: bootstrap.answerSdp
1529
+ });
1530
+ return true;
1531
+ }
1532
+ shouldRebuildConnection() {
1533
+ return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
1534
+ }
1535
+ async rebuildConnection(config) {
1536
+ this.connectionLossNotified = true;
1537
+ this.cleanup();
1538
+ await this.connect(config ?? this.currentSessionConfig);
1539
+ }
1034
1540
  resolvePendingSessionUpdate() {
1035
1541
  if (!this.pendingSessionUpdate) {
1036
1542
  return;
@@ -1105,6 +1611,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
1105
1611
  }
1106
1612
  return session;
1107
1613
  }
1614
+ function delay(ms) {
1615
+ return new Promise((resolve) => {
1616
+ setTimeout(resolve, ms);
1617
+ });
1618
+ }
1108
1619
 
1109
1620
  // src/remote/index.ts
1110
1621
  var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
@@ -1131,30 +1642,33 @@ var RemoteRealtimeClient = class {
1131
1642
  adapters;
1132
1643
  eventCallbacks = /* @__PURE__ */ new Set();
1133
1644
  async connect(config) {
1134
- const adapterId = this.resolveAdapterId(config);
1135
- const factory = this.adapters.get(adapterId);
1136
- if (!factory) {
1137
- throw new Error(
1138
- `No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
1139
- );
1140
- }
1141
- const transportClient = factory({
1142
- debug: this.config.debug,
1143
- requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
1144
- });
1145
- for (const callback of this.eventCallbacks) {
1146
- transportClient.onEvent(callback);
1147
- }
1148
- this.transportClient = transportClient;
1149
1645
  try {
1646
+ const adapterId = this.resolveAdapterId(config);
1647
+ const factory = this.adapters.get(adapterId);
1648
+ if (!factory) {
1649
+ throw new core.CharivoStateError(
1650
+ `No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
1651
+ );
1652
+ }
1653
+ const transportClient = factory({
1654
+ debug: this.config.debug,
1655
+ requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
1656
+ });
1657
+ for (const callback of this.eventCallbacks) {
1658
+ transportClient.onEvent(callback);
1659
+ }
1660
+ this.transportClient = transportClient;
1150
1661
  await transportClient.connect(config);
1151
1662
  } catch (error) {
1152
1663
  this.transportClient = null;
1153
- throw error;
1664
+ throw core.toCharivoError("transport", error);
1154
1665
  }
1155
1666
  }
1156
1667
  async updateSession(config) {
1157
- await this.getActiveTransportClient().updateSession(config);
1668
+ await this.getActiveTransportClient().updateSession(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1669
+ }
1670
+ async recover(config) {
1671
+ await this.getActiveTransportClient().recover(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1158
1672
  }
1159
1673
  async disconnect() {
1160
1674
  if (!this.transportClient) {
@@ -1162,19 +1676,19 @@ var RemoteRealtimeClient = class {
1162
1676
  }
1163
1677
  const transportClient = this.transportClient;
1164
1678
  this.transportClient = null;
1165
- await transportClient.disconnect();
1679
+ await transportClient.disconnect().catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1166
1680
  }
1167
1681
  async sendText(text) {
1168
- await this.getActiveTransportClient().sendText(text);
1682
+ await this.getActiveTransportClient().sendText(text).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1169
1683
  }
1170
1684
  async sendAudio(audio) {
1171
- await this.getActiveTransportClient().sendAudio(audio);
1685
+ await this.getActiveTransportClient().sendAudio(audio).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1172
1686
  }
1173
1687
  async sendToolResult(callId, output) {
1174
- await this.getActiveTransportClient().sendToolResult(callId, output);
1688
+ await this.getActiveTransportClient().sendToolResult(callId, output).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1175
1689
  }
1176
1690
  async interrupt() {
1177
- await this.getActiveTransportClient().interrupt();
1691
+ await this.getActiveTransportClient().interrupt().catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1178
1692
  }
1179
1693
  onEvent(callback) {
1180
1694
  this.eventCallbacks.add(callback);
@@ -1182,7 +1696,7 @@ var RemoteRealtimeClient = class {
1182
1696
  }
1183
1697
  getActiveTransportClient() {
1184
1698
  if (!this.transportClient) {
1185
- throw new Error("Realtime session not active");
1699
+ throw new core.CharivoStateError("Realtime session not active");
1186
1700
  }
1187
1701
  return this.transportClient;
1188
1702
  }
@@ -1194,7 +1708,7 @@ var RemoteRealtimeClient = class {
1194
1708
  if (config?.provider === "openai" && transport === "webrtc") {
1195
1709
  return core.OPENAI_REALTIME_AGENTS_ADAPTER;
1196
1710
  }
1197
- throw new Error(
1711
+ throw new core.CharivoStateError(
1198
1712
  `No remote realtime adapter resolver for provider "${config?.provider ?? "(unspecified)"}" and transport "${transport}"`
1199
1713
  );
1200
1714
  }
@@ -1215,14 +1729,18 @@ var RemoteRealtimeClient = class {
1215
1729
  );
1216
1730
  if (!response.ok) {
1217
1731
  const errorText = await response.text();
1218
- throw new Error(`Failed to create Realtime session: ${errorText}`);
1732
+ throw new core.CharivoProviderError(
1733
+ `Failed to create Realtime session: ${errorText}`
1734
+ );
1219
1735
  }
1220
1736
  const bootstrap = await response.json();
1221
1737
  if (!isRealtimeSessionBootstrap(bootstrap)) {
1222
- throw new Error("Invalid realtime session bootstrap response");
1738
+ throw new core.CharivoProviderError(
1739
+ "Invalid realtime session bootstrap response"
1740
+ );
1223
1741
  }
1224
1742
  if (bootstrap.adapter !== options.expectedAdapterId) {
1225
- throw new Error(
1743
+ throw new core.CharivoProviderError(
1226
1744
  `Realtime session bootstrap adapter mismatch: expected ${options.expectedAdapterId}, received ${bootstrap.adapter}`
1227
1745
  );
1228
1746
  }