@charivo/realtime 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-WLTIMNHQ.mjs → chunk-DHNRDHST.mjs} +248 -13
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/{chunk-FABRM56W.mjs → chunk-PTFZ7CML.mjs} +208 -12
  5. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  6. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  7. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +217 -3
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +214 -3
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +323 -10
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +283 -9
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +531 -19
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +6 -3
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/openai-agents/client.ts
7
7
 
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
85
+
8
86
  // src/internal/shared.ts
9
87
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
88
  function isRecord(value) {
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
83
161
  }
84
162
  audioAnalysisStream = null;
85
163
  audioContext = null;
164
+ audioSource = null;
86
165
  analyser = null;
87
166
  lipSyncInterval = null;
88
167
  pendingAudioElementPoll = null;
89
168
  observedAudioElement = null;
90
169
  observedAudioElementListener = null;
170
+ async prepareAudioContext() {
171
+ if (this.audioContext) {
172
+ return;
173
+ }
174
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
175
+ if (!audioContextConstructor) {
176
+ throw new Error("AudioContext is not supported in this browser");
177
+ }
178
+ this.audioContext = new audioContextConstructor();
179
+ }
91
180
  observeAudioElement(audioElement) {
92
181
  this.detachObservedAudioElement();
93
182
  const tryAttachStream = () => {
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
109
198
  this.stopLipSyncAnalysis();
110
199
  this.audioAnalysisStream = stream;
111
200
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
201
+ if (!this.audioContext) {
202
+ void this.prepareAudioContext().catch((error) => {
203
+ this.options.onError?.(error);
204
+ });
205
+ }
206
+ if (!this.audioContext) {
207
+ return;
115
208
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
209
+ this.audioSource?.disconnect?.();
210
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
211
  this.analyser = this.audioContext.createAnalyser();
119
212
  this.analyser.fftSize = 256;
120
213
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
214
+ this.audioSource.connect(this.analyser);
122
215
  this.startLipSyncAnalysis();
123
216
  } catch (error) {
124
217
  this.options.onError?.(error);
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
127
220
  stopOutput() {
128
221
  this.stopLipSyncAnalysis();
129
222
  }
223
+ pause() {
224
+ this.stopLipSyncAnalysis();
225
+ }
226
+ resume() {
227
+ if (!this.analyser || this.lipSyncInterval) {
228
+ return;
229
+ }
230
+ this.startLipSyncAnalysis();
231
+ }
130
232
  cleanup() {
131
233
  this.stopLipSyncAnalysis();
132
234
  this.stopAudioElementPolling();
133
235
  this.detachObservedAudioElement();
236
+ this.audioSource?.disconnect?.();
237
+ this.audioSource = null;
134
238
  if (this.audioContext) {
135
239
  void this.audioContext.close();
136
240
  this.audioContext = null;
@@ -265,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
265
369
  session = null;
266
370
  transport = null;
267
371
  audioElement = null;
372
+ mediaStream = null;
373
+ peerConnection = null;
374
+ audioSender = null;
268
375
  connectionWasActive = false;
376
+ connectionLossNotified = false;
377
+ isExplicitDisconnect = false;
378
+ isRecovering = false;
379
+ isCleaningUp = false;
269
380
  assistant = { text: "", started: false };
270
381
  latestHistory = [];
382
+ currentSessionConfig;
383
+ teardownTransportLifecycle;
384
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
385
+ this.emitConnectionLost("ice-disconnected");
386
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
271
387
  eventCallbacks = /* @__PURE__ */ new Set();
272
388
  pendingToolCalls = /* @__PURE__ */ new Map();
273
389
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -280,13 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
280
396
  });
281
397
  async connect(config) {
282
398
  try {
399
+ if (this.isExplicitDisconnect && this.isRecovering) {
400
+ throw new Error("Realtime session disconnect requested");
401
+ }
283
402
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
403
+ this.isExplicitDisconnect = false;
404
+ this.isRecovering = false;
405
+ this.connectionLossNotified = false;
406
+ this.currentSessionConfig = config;
284
407
  this.audioElement = document.createElement("audio");
285
408
  this.audioElement.autoplay = true;
286
409
  this.audioElement.setAttribute("playsinline", "true");
410
+ this.mediaStream = await acquireMicrophoneStream();
411
+ this.bindTransportLifecycleEvents();
412
+ await this.lipSyncAnalyzer.prepareAudioContext();
287
413
  const agent = this.createAgent(config);
288
414
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
289
- this.createTransportOptions(this.audioElement)
415
+ this.createTransportOptions(this.audioElement, this.mediaStream)
290
416
  );
291
417
  this.session = new agentsRealtime.RealtimeSession(agent, {
292
418
  transport: this.transport,
@@ -318,6 +444,8 @@ var OpenAIRealtimeAgentsClient = class {
318
444
  }
319
445
  async disconnect() {
320
446
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
447
+ this.isExplicitDisconnect = true;
448
+ this.isRecovering = false;
321
449
  this.cleanup();
322
450
  }
323
451
  async updateSession(config) {
@@ -326,8 +454,27 @@ var OpenAIRealtimeAgentsClient = class {
326
454
  }
327
455
  this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
328
456
  await this.session.updateAgent(this.createAgent(config));
457
+ this.currentSessionConfig = config;
458
+ }
459
+ async recover(config) {
460
+ this.isRecovering = true;
461
+ this.connectionLossNotified = true;
462
+ this.currentSessionConfig = config;
463
+ try {
464
+ this.cleanupPendingToolCalls(
465
+ new Error("Realtime session interrupted during reconnect")
466
+ );
467
+ this.cleanup();
468
+ await this.connect(config ?? this.currentSessionConfig);
469
+ this.connectionLossNotified = false;
470
+ } finally {
471
+ this.isRecovering = false;
472
+ }
329
473
  }
330
474
  async sendText(text) {
475
+ if (this.isRecovering) {
476
+ throw new Error("Realtime transport reconnecting");
477
+ }
331
478
  if (!this.session) {
332
479
  throw new Error("Realtime session not active");
333
480
  }
@@ -335,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
335
482
  this.session.sendMessage(text);
336
483
  }
337
484
  async sendAudio(_audio) {
485
+ if (this.isRecovering) {
486
+ throw new Error("Realtime transport reconnecting");
487
+ }
338
488
  console.warn(
339
489
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
340
490
  );
341
491
  }
342
492
  async sendToolResult(callId, output) {
493
+ if (this.isRecovering) {
494
+ throw new Error("Realtime transport reconnecting");
495
+ }
343
496
  const pendingCall = this.pendingToolCalls.get(callId);
344
497
  if (!pendingCall) {
345
498
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -348,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
348
501
  pendingCall.resolve(output);
349
502
  }
350
503
  async interrupt() {
504
+ if (this.isRecovering) {
505
+ throw new Error("Realtime transport reconnecting");
506
+ }
351
507
  if (!this.session) {
352
508
  throw new Error("Realtime session not active");
353
509
  }
@@ -411,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
411
567
  if (status === "disconnected" && this.connectionWasActive) {
412
568
  this.cleanupPendingToolCalls();
413
569
  this.connectionWasActive = false;
414
- this.emitEvent({ type: "session.ended" });
570
+ this.emitConnectionLost("connection-failed");
415
571
  }
416
572
  });
417
573
  }
@@ -513,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
513
669
  });
514
670
  });
515
671
  }
516
- createTransportOptions(audioElement) {
672
+ createTransportOptions(audioElement, mediaStream) {
517
673
  return {
518
674
  audioElement,
675
+ mediaStream,
519
676
  changePeerConnection: async (peerConnection) => {
677
+ this.peerConnection = peerConnection;
678
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
679
+ this.bindPeerConnectionEvents(peerConnection);
520
680
  peerConnection.addEventListener("track", (event) => {
521
681
  const stream = event.streams[0];
522
682
  if (stream) {
@@ -527,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
527
687
  }
528
688
  };
529
689
  }
690
+ bindPeerConnectionEvents(peerConnection) {
691
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
692
+ const iceState = peerConnection.iceConnectionState;
693
+ if (iceState === "failed") {
694
+ this.iceDisconnectDebouncer.cancel();
695
+ this.emitConnectionLost("ice-failed");
696
+ } else if (iceState === "disconnected") {
697
+ this.iceDisconnectDebouncer.schedule();
698
+ } else {
699
+ this.iceDisconnectDebouncer.cancel();
700
+ }
701
+ });
702
+ peerConnection.addEventListener("connectionstatechange", () => {
703
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
704
+ this.iceDisconnectDebouncer.cancel();
705
+ this.emitConnectionLost("connection-failed");
706
+ }
707
+ });
708
+ }
709
+ bindTransportLifecycleEvents() {
710
+ if (this.teardownTransportLifecycle) {
711
+ return;
712
+ }
713
+ this.teardownTransportLifecycle = bindTransportLifecycle({
714
+ onHidden: this.handleHidden,
715
+ onOnline: this.handleOnline,
716
+ onPageHide: this.handlePageHide,
717
+ onPageShow: this.handlePageShow,
718
+ onVisible: this.handleVisible,
719
+ onDeviceChange: this.handleDeviceChange
720
+ });
721
+ }
722
+ unbindTransportLifecycleEvents() {
723
+ this.teardownTransportLifecycle?.();
724
+ this.teardownTransportLifecycle = void 0;
725
+ }
726
+ handleOnline = () => {
727
+ this.emitConnectionLost("online");
728
+ };
729
+ handleHidden = () => {
730
+ this.lipSyncAnalyzer.pause();
731
+ };
732
+ handleVisible = () => {
733
+ this.lipSyncAnalyzer.resume();
734
+ this.emitConnectionLost("visibility");
735
+ };
736
+ handlePageHide = () => {
737
+ this.lipSyncAnalyzer.pause();
738
+ };
739
+ handlePageShow = (event) => {
740
+ this.lipSyncAnalyzer.resume();
741
+ if (event.persisted) {
742
+ this.emitConnectionLost("pageshow");
743
+ }
744
+ };
745
+ handleDeviceChange = () => {
746
+ void this.refreshMicrophoneTrack();
747
+ };
748
+ async refreshMicrophoneTrack() {
749
+ if (!this.peerConnection || this.isExplicitDisconnect) {
750
+ return;
751
+ }
752
+ try {
753
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
754
+ audioSender: this.audioSender,
755
+ mediaStream: this.mediaStream,
756
+ peerConnection: this.peerConnection
757
+ });
758
+ this.audioSender = audioSender;
759
+ this.mediaStream = mediaStream;
760
+ } catch (error) {
761
+ this.emitEvent({
762
+ type: "error",
763
+ error: error instanceof Error ? error : new Error(String(error))
764
+ });
765
+ }
766
+ }
767
+ emitConnectionLost(cause, error) {
768
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
769
+ return;
770
+ }
771
+ this.connectionLossNotified = true;
772
+ this.cleanupPendingToolCalls(
773
+ new Error("Realtime session interrupted during reconnect")
774
+ );
775
+ this.resetAssistantTracking();
776
+ this.lipSyncAnalyzer.stopOutput();
777
+ this.emitEvent({
778
+ type: "connection.lost",
779
+ cause,
780
+ error
781
+ });
782
+ }
783
+ shouldRecover(cause) {
784
+ if (!this.peerConnection) {
785
+ return false;
786
+ }
787
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
788
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
789
+ }
790
+ return true;
791
+ }
530
792
  cleanup(error) {
793
+ this.connectionWasActive = false;
794
+ this.isCleaningUp = true;
795
+ this.iceDisconnectDebouncer.cancel();
796
+ this.unbindTransportLifecycleEvents();
531
797
  this.cleanupPendingToolCalls(error);
532
798
  this.lipSyncAnalyzer.cleanup();
533
799
  if (this.transport) {
@@ -542,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
542
808
  this.audioElement.srcObject = null;
543
809
  this.audioElement = null;
544
810
  }
811
+ if (this.mediaStream) {
812
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
813
+ this.mediaStream = null;
814
+ }
815
+ this.peerConnection = null;
816
+ this.audioSender = null;
545
817
  this.latestHistory = [];
818
+ this.connectionLossNotified = false;
819
+ this.isCleaningUp = false;
546
820
  this.resetAssistantTracking();
547
821
  }
548
822
  resetAssistantTracking() {