@charivo/realtime 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-WLTIMNHQ.mjs → chunk-DHNRDHST.mjs} +248 -13
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/{chunk-FABRM56W.mjs → chunk-PTFZ7CML.mjs} +208 -12
  5. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  6. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  7. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +217 -3
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +214 -3
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +323 -10
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +283 -9
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +531 -19
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +6 -3
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -7,6 +7,84 @@ var core = require('@charivo/core');
7
7
  // src/instructions.ts
8
8
  var DEFAULT_REALTIME_VOICE = "marin";
9
9
 
10
+ // src/internal/microphone.ts
11
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
12
+ echoCancellation: true,
13
+ noiseSuppression: true,
14
+ autoGainControl: true
15
+ };
16
+ async function acquireMicrophoneStream() {
17
+ return navigator.mediaDevices.getUserMedia({
18
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
19
+ });
20
+ }
21
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
22
+ function bindTransportLifecycle(callbacks) {
23
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
24
+ onHidden: callbacks.onHidden,
25
+ onOnline: callbacks.onOnline,
26
+ onPageHide: callbacks.onPageHide,
27
+ onPageShow: callbacks.onPageShow,
28
+ onVisible: callbacks.onVisible
29
+ });
30
+ if (callbacks.onDeviceChange) {
31
+ navigator.mediaDevices?.addEventListener?.(
32
+ "devicechange",
33
+ callbacks.onDeviceChange
34
+ );
35
+ }
36
+ return () => {
37
+ teardownBrowserLifecycle();
38
+ if (callbacks.onDeviceChange) {
39
+ navigator.mediaDevices?.removeEventListener?.(
40
+ "devicechange",
41
+ callbacks.onDeviceChange
42
+ );
43
+ }
44
+ };
45
+ }
46
+ async function replaceMicrophoneTrack(state) {
47
+ if (!state.peerConnection) {
48
+ throw new Error("Realtime transport not active");
49
+ }
50
+ const nextStream = await acquireMicrophoneStream();
51
+ const nextTrack = nextStream.getAudioTracks()[0];
52
+ if (!nextTrack) {
53
+ throw new Error("Microphone access required for Realtime API");
54
+ }
55
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
56
+ if (!sender) {
57
+ throw new Error("No outbound audio sender available");
58
+ }
59
+ await sender.replaceTrack(nextTrack);
60
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
61
+ return {
62
+ audioSender: sender,
63
+ mediaStream: nextStream
64
+ };
65
+ }
66
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
67
+ let pending = null;
68
+ return {
69
+ schedule() {
70
+ if (pending) {
71
+ return;
72
+ }
73
+ pending = setTimeout(() => {
74
+ pending = null;
75
+ onDisconnect();
76
+ }, delayMs);
77
+ },
78
+ cancel() {
79
+ if (!pending) {
80
+ return;
81
+ }
82
+ clearTimeout(pending);
83
+ pending = null;
84
+ }
85
+ };
86
+ }
87
+
10
88
  // src/internal/shared.ts
11
89
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
12
90
  function isRecord(value) {
@@ -64,7 +142,9 @@ var OpenAIRealtimeClient = class {
64
142
  dc = null;
65
143
  audioElement = null;
66
144
  mediaStream = null;
145
+ audioSender = null;
67
146
  audioContext = null;
147
+ audioSource = null;
68
148
  analyser = null;
69
149
  lipSyncInterval = null;
70
150
  isResponseInProgress = false;
@@ -74,13 +154,33 @@ var OpenAIRealtimeClient = class {
74
154
  assistantText = "";
75
155
  nextSessionUpdateId = 1;
76
156
  pendingSessionUpdate = null;
157
+ currentSessionConfig;
158
+ connectionLossNotified = false;
159
+ isExplicitDisconnect = false;
160
+ isRecovering = false;
161
+ isCleaningUp = false;
162
+ teardownTransportLifecycle;
163
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
164
+ this.emitConnectionLost("ice-disconnected");
165
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
77
166
  eventCallbacks = /* @__PURE__ */ new Set();
78
167
  async connect(config) {
79
168
  try {
169
+ if (this.isExplicitDisconnect && this.isRecovering) {
170
+ throw new Error("Realtime session disconnect requested");
171
+ }
80
172
  this.log("Starting OpenAI Realtime WebRTC connection");
173
+ this.isExplicitDisconnect = false;
174
+ this.isRecovering = false;
175
+ this.connectionLossNotified = false;
176
+ this.currentSessionConfig = config;
81
177
  this.pc = new RTCPeerConnection();
82
178
  this.audioElement = document.createElement("audio");
83
179
  this.audioElement.autoplay = true;
180
+ this.audioElement.setAttribute("playsinline", "true");
181
+ this.prepareAudioAnalysis();
182
+ this.bindConnectionEvents();
183
+ this.bindTransportLifecycleEvents();
84
184
  this.pc.ontrack = (event) => {
85
185
  if (this.audioElement) {
86
186
  this.audioElement.srcObject = event.streams[0];
@@ -88,11 +188,9 @@ var OpenAIRealtimeClient = class {
88
188
  this.setupAudioAnalysis(event.streams[0]);
89
189
  };
90
190
  try {
91
- this.mediaStream = await navigator.mediaDevices.getUserMedia({
92
- audio: true
93
- });
191
+ this.mediaStream = await acquireMicrophoneStream();
94
192
  const audioTrack = this.mediaStream.getTracks()[0];
95
- this.pc.addTrack(audioTrack, this.mediaStream);
193
+ this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
96
194
  } catch {
97
195
  throw new Error("Microphone access required for Realtime API");
98
196
  }
@@ -131,6 +229,8 @@ var OpenAIRealtimeClient = class {
131
229
  }
132
230
  async disconnect() {
133
231
  this.log("Disconnecting OpenAI Realtime WebRTC");
232
+ this.isExplicitDisconnect = true;
233
+ this.isRecovering = false;
134
234
  this.cleanup();
135
235
  }
136
236
  async updateSession(config) {
@@ -166,8 +266,27 @@ var OpenAIRealtimeClient = class {
166
266
  timeoutId
167
267
  };
168
268
  });
269
+ this.currentSessionConfig = config;
270
+ }
271
+ async recover(config) {
272
+ this.isRecovering = true;
273
+ this.connectionLossNotified = true;
274
+ this.currentSessionConfig = config;
275
+ try {
276
+ if (await this.tryIceRestartRecovery(config)) {
277
+ this.connectionLossNotified = false;
278
+ return;
279
+ }
280
+ await this.rebuildConnection(config);
281
+ this.connectionLossNotified = false;
282
+ } finally {
283
+ this.isRecovering = false;
284
+ }
169
285
  }
170
286
  async sendText(text) {
287
+ if (this.isRecovering) {
288
+ throw new Error("Realtime transport reconnecting");
289
+ }
171
290
  if (!this.dc || this.dc.readyState !== "open") {
172
291
  throw new Error("DataChannel not ready");
173
292
  }
@@ -194,11 +313,17 @@ var OpenAIRealtimeClient = class {
194
313
  this.beginResponseRequest();
195
314
  }
196
315
  async sendAudio(_audio) {
316
+ if (this.isRecovering) {
317
+ throw new Error("Realtime transport reconnecting");
318
+ }
197
319
  console.warn(
198
320
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
199
321
  );
200
322
  }
201
323
  async sendToolResult(callId, output) {
324
+ if (this.isRecovering) {
325
+ throw new Error("Realtime transport reconnecting");
326
+ }
202
327
  const dc = this.requireOpenDataChannel();
203
328
  dc.send(
204
329
  JSON.stringify({
@@ -214,6 +339,9 @@ var OpenAIRealtimeClient = class {
214
339
  this.beginResponseRequest();
215
340
  }
216
341
  async interrupt() {
342
+ if (this.isRecovering) {
343
+ throw new Error("Realtime transport reconnecting");
344
+ }
217
345
  const dc = this.requireOpenDataChannel();
218
346
  if (!this.isResponseInProgress) {
219
347
  return;
@@ -410,18 +538,133 @@ var OpenAIRealtimeClient = class {
410
538
  }
411
539
  return bootstrap;
412
540
  }
541
+ bindConnectionEvents() {
542
+ if (!this.pc) {
543
+ return;
544
+ }
545
+ this.pc.addEventListener("iceconnectionstatechange", () => {
546
+ const iceState = this.pc?.iceConnectionState;
547
+ if (iceState === "failed") {
548
+ this.iceDisconnectDebouncer.cancel();
549
+ this.emitConnectionLost("ice-failed");
550
+ } else if (iceState === "disconnected") {
551
+ this.iceDisconnectDebouncer.schedule();
552
+ } else {
553
+ this.iceDisconnectDebouncer.cancel();
554
+ }
555
+ });
556
+ this.pc.addEventListener("connectionstatechange", () => {
557
+ const connectionState = this.pc?.connectionState;
558
+ if (connectionState === "failed" || connectionState === "closed") {
559
+ this.iceDisconnectDebouncer.cancel();
560
+ this.emitConnectionLost("connection-failed");
561
+ }
562
+ });
563
+ }
564
+ bindTransportLifecycleEvents() {
565
+ if (this.teardownTransportLifecycle) {
566
+ return;
567
+ }
568
+ this.teardownTransportLifecycle = bindTransportLifecycle({
569
+ onHidden: this.handleHidden,
570
+ onOnline: this.handleOnline,
571
+ onPageHide: this.handlePageHide,
572
+ onPageShow: this.handlePageShow,
573
+ onVisible: this.handleVisible,
574
+ onDeviceChange: this.handleDeviceChange
575
+ });
576
+ }
577
+ unbindTransportLifecycleEvents() {
578
+ this.teardownTransportLifecycle?.();
579
+ this.teardownTransportLifecycle = void 0;
580
+ }
581
+ handleOnline = () => {
582
+ this.emitConnectionLost("online");
583
+ };
584
+ handleHidden = () => {
585
+ this.pauseAudioAnalysis();
586
+ };
587
+ handleVisible = () => {
588
+ this.resumeAudioAnalysis();
589
+ this.emitConnectionLost("visibility");
590
+ };
591
+ handlePageHide = () => {
592
+ this.pauseAudioAnalysis();
593
+ };
594
+ handlePageShow = (event) => {
595
+ this.resumeAudioAnalysis();
596
+ if (event.persisted) {
597
+ this.emitConnectionLost("pageshow");
598
+ }
599
+ };
600
+ handleDeviceChange = () => {
601
+ void this.refreshMicrophoneTrack();
602
+ };
603
+ async refreshMicrophoneTrack() {
604
+ if (!this.pc || this.isExplicitDisconnect) {
605
+ return;
606
+ }
607
+ try {
608
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
609
+ audioSender: this.audioSender,
610
+ mediaStream: this.mediaStream,
611
+ peerConnection: this.pc
612
+ });
613
+ this.audioSender = audioSender;
614
+ this.mediaStream = mediaStream;
615
+ } catch (error) {
616
+ this.emitEvent({
617
+ type: "error",
618
+ error: error instanceof Error ? error : new Error(String(error))
619
+ });
620
+ }
621
+ }
622
+ emitConnectionLost(cause, error) {
623
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
624
+ return;
625
+ }
626
+ this.connectionLossNotified = true;
627
+ this.isResponseInProgress = false;
628
+ this.cancelInFlight = false;
629
+ this.resetResponseTracking();
630
+ this.stopLipSyncAnalysis();
631
+ this.emitEvent({
632
+ type: "connection.lost",
633
+ cause,
634
+ error
635
+ });
636
+ }
637
+ needsRecovery(cause) {
638
+ if (!this.pc || !this.dc) {
639
+ return false;
640
+ }
641
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
642
+ return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
643
+ }
644
+ return true;
645
+ }
646
+ prepareAudioAnalysis() {
647
+ if (this.audioContext) {
648
+ return;
649
+ }
650
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
651
+ if (!audioContextConstructor) {
652
+ return;
653
+ }
654
+ this.audioContext = new audioContextConstructor();
655
+ }
413
656
  setupAudioAnalysis(stream) {
414
657
  try {
415
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
416
- if (!audioContextConstructor) {
417
- throw new Error("AudioContext is not supported in this browser");
658
+ this.prepareAudioAnalysis();
659
+ if (!this.audioContext) {
660
+ return;
418
661
  }
419
- this.audioContext = new audioContextConstructor();
420
- const source = this.audioContext.createMediaStreamSource(stream);
662
+ this.audioSource?.disconnect();
663
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
421
664
  this.analyser = this.audioContext.createAnalyser();
422
665
  this.analyser.fftSize = 256;
423
666
  this.analyser.smoothingTimeConstant = 0.8;
424
- source.connect(this.analyser);
667
+ this.audioSource.connect(this.analyser);
425
668
  this.startLipSyncAnalysis();
426
669
  } catch (error) {
427
670
  console.error("Failed to setup audio analysis:", error);
@@ -457,7 +700,19 @@ var OpenAIRealtimeClient = class {
457
700
  }
458
701
  this.emitEvent({ type: "audio.lipsync", rms: 0 });
459
702
  }
703
+ pauseAudioAnalysis() {
704
+ this.stopLipSyncAnalysis();
705
+ }
706
+ resumeAudioAnalysis() {
707
+ if (!this.analyser || this.lipSyncInterval) {
708
+ return;
709
+ }
710
+ this.startLipSyncAnalysis();
711
+ }
460
712
  cleanup() {
713
+ this.isCleaningUp = true;
714
+ this.unbindTransportLifecycleEvents();
715
+ this.iceDisconnectDebouncer.cancel();
461
716
  this.stopLipSyncAnalysis();
462
717
  this.rejectPendingSessionUpdate(
463
718
  new Error("Realtime session ended before session update completed")
@@ -482,9 +737,13 @@ var OpenAIRealtimeClient = class {
482
737
  this.audioContext.close();
483
738
  this.audioContext = null;
484
739
  }
740
+ this.audioSource = null;
741
+ this.audioSender = null;
485
742
  this.analyser = null;
486
743
  this.isResponseInProgress = false;
487
744
  this.cancelInFlight = false;
745
+ this.connectionLossNotified = false;
746
+ this.isCleaningUp = false;
488
747
  this.resetResponseTracking();
489
748
  }
490
749
  beginResponseRequest() {
@@ -493,11 +752,60 @@ var OpenAIRealtimeClient = class {
493
752
  this.cancelInFlight = false;
494
753
  }
495
754
  requireOpenDataChannel() {
755
+ if (this.isRecovering) {
756
+ throw new Error("Realtime transport reconnecting");
757
+ }
496
758
  if (!this.dc || this.dc.readyState !== "open") {
497
759
  throw new Error("DataChannel not ready");
498
760
  }
499
761
  return this.dc;
500
762
  }
763
+ async tryIceRestartRecovery(config) {
764
+ const pc = this.pc;
765
+ if (!pc || this.shouldRebuildConnection()) {
766
+ return false;
767
+ }
768
+ if (pc.iceConnectionState === "connected") {
769
+ return true;
770
+ }
771
+ if (pc.iceConnectionState === "disconnected") {
772
+ await delay(1500);
773
+ if (pc.iceConnectionState !== "disconnected") {
774
+ return true;
775
+ }
776
+ }
777
+ if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
778
+ return false;
779
+ }
780
+ if (typeof pc.restartIce === "function") {
781
+ pc.restartIce();
782
+ }
783
+ const offer = await pc.createOffer({ iceRestart: true });
784
+ await pc.setLocalDescription(offer);
785
+ const bootstrap = await this.getSessionBootstrap({
786
+ transport: "webrtc",
787
+ session: config ?? this.currentSessionConfig ?? {},
788
+ sdpOffer: offer.sdp
789
+ });
790
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
791
+ throw new Error(
792
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
793
+ );
794
+ }
795
+ await pc.setRemoteDescription({
796
+ type: "answer",
797
+ sdp: bootstrap.answerSdp
798
+ });
799
+ return true;
800
+ }
801
+ shouldRebuildConnection() {
802
+ return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
803
+ }
804
+ async rebuildConnection(config) {
805
+ this.connectionLossNotified = true;
806
+ this.cleanup();
807
+ await this.connect(config ?? this.currentSessionConfig);
808
+ }
501
809
  resolvePendingSessionUpdate() {
502
810
  if (!this.pendingSessionUpdate) {
503
811
  return;
@@ -572,6 +880,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
572
880
  }
573
881
  return session;
574
882
  }
883
+ function delay(ms) {
884
+ return new Promise((resolve) => {
885
+ setTimeout(resolve, ms);
886
+ });
887
+ }
575
888
 
576
889
  exports.OpenAIRealtimeClient = OpenAIRealtimeClient;
577
890
  exports.createOpenAIRealtimeClient = createOpenAIRealtimeClient;