@charivo/realtime 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-FABRM56W.mjs → chunk-3O3COMUY.mjs} +216 -17
  3. package/dist/chunk-3O3COMUY.mjs.map +1 -0
  4. package/dist/{chunk-WLTIMNHQ.mjs → chunk-J5L3BPMG.mjs} +248 -13
  5. package/dist/chunk-J5L3BPMG.mjs.map +1 -0
  6. package/dist/chunk-ZJZO4QYA.mjs +123 -0
  7. package/dist/chunk-ZJZO4QYA.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +240 -21
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +237 -21
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +326 -13
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +293 -17
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +572 -54
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +38 -31
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,7 +5,83 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/openai-agents/client.ts
7
7
 
8
- // src/internal/shared.ts
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
9
85
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
86
  function isRecord(value) {
11
87
  return typeof value === "object" && value !== null;
@@ -32,9 +108,11 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
32
108
  });
33
109
  } catch (error) {
34
110
  if (isAbortError(error)) {
35
- throw new Error(timeoutMessage);
111
+ throw new core.CharivoTimeoutError(timeoutMessage, { cause: error });
36
112
  }
37
- throw error;
113
+ throw new core.CharivoTransportError("Realtime request failed", {
114
+ cause: error instanceof Error ? error : void 0
115
+ });
38
116
  } finally {
39
117
  clearTimeout(timeoutId);
40
118
  }
@@ -42,15 +120,13 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
42
120
  function isAbortError(error) {
43
121
  return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
44
122
  }
45
-
46
- // src/openai-agents/bootstrap.ts
47
123
  async function getOpenAIRealtimeAgentsBootstrap(options, request) {
48
124
  if (options.sessionBootstrap) {
49
125
  return options.sessionBootstrap(request);
50
126
  }
51
127
  const apiEndpoint = options.apiEndpoint;
52
128
  if (!apiEndpoint) {
53
- throw new Error(
129
+ throw new core.CharivoStateError(
54
130
  "OpenAI agents realtime client requires apiEndpoint or sessionBootstrap"
55
131
  );
56
132
  }
@@ -67,11 +143,15 @@ async function getOpenAIRealtimeAgentsBootstrap(options, request) {
67
143
  );
68
144
  if (!response.ok) {
69
145
  const errorText = await response.text();
70
- throw new Error(`Failed to create Realtime session: ${errorText}`);
146
+ throw new core.CharivoProviderError(
147
+ `Failed to create Realtime session: ${errorText}`
148
+ );
71
149
  }
72
150
  const bootstrap = await response.json();
73
151
  if (!isRealtimeSessionBootstrap(bootstrap)) {
74
- throw new Error("Invalid realtime session bootstrap response");
152
+ throw new core.CharivoProviderError(
153
+ "Invalid realtime session bootstrap response"
154
+ );
75
155
  }
76
156
  return bootstrap;
77
157
  }
@@ -83,11 +163,22 @@ var LipSyncAnalyzer = class {
83
163
  }
84
164
  audioAnalysisStream = null;
85
165
  audioContext = null;
166
+ audioSource = null;
86
167
  analyser = null;
87
168
  lipSyncInterval = null;
88
169
  pendingAudioElementPoll = null;
89
170
  observedAudioElement = null;
90
171
  observedAudioElementListener = null;
172
+ async prepareAudioContext() {
173
+ if (this.audioContext) {
174
+ return;
175
+ }
176
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
177
+ if (!audioContextConstructor) {
178
+ throw new Error("AudioContext is not supported in this browser");
179
+ }
180
+ this.audioContext = new audioContextConstructor();
181
+ }
91
182
  observeAudioElement(audioElement) {
92
183
  this.detachObservedAudioElement();
93
184
  const tryAttachStream = () => {
@@ -109,16 +200,20 @@ var LipSyncAnalyzer = class {
109
200
  this.stopLipSyncAnalysis();
110
201
  this.audioAnalysisStream = stream;
111
202
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
203
+ if (!this.audioContext) {
204
+ void this.prepareAudioContext().catch((error) => {
205
+ this.options.onError?.(error);
206
+ });
207
+ }
208
+ if (!this.audioContext) {
209
+ return;
115
210
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
211
+ this.audioSource?.disconnect?.();
212
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
213
  this.analyser = this.audioContext.createAnalyser();
119
214
  this.analyser.fftSize = 256;
120
215
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
216
+ this.audioSource.connect(this.analyser);
122
217
  this.startLipSyncAnalysis();
123
218
  } catch (error) {
124
219
  this.options.onError?.(error);
@@ -127,10 +222,21 @@ var LipSyncAnalyzer = class {
127
222
  stopOutput() {
128
223
  this.stopLipSyncAnalysis();
129
224
  }
225
+ pause() {
226
+ this.stopLipSyncAnalysis();
227
+ }
228
+ resume() {
229
+ if (!this.analyser || this.lipSyncInterval) {
230
+ return;
231
+ }
232
+ this.startLipSyncAnalysis();
233
+ }
130
234
  cleanup() {
131
235
  this.stopLipSyncAnalysis();
132
236
  this.stopAudioElementPolling();
133
237
  this.detachObservedAudioElement();
238
+ this.audioSource?.disconnect?.();
239
+ this.audioSource = null;
134
240
  if (this.audioContext) {
135
241
  void this.audioContext.close();
136
242
  this.audioContext = null;
@@ -265,9 +371,21 @@ var OpenAIRealtimeAgentsClient = class {
265
371
  session = null;
266
372
  transport = null;
267
373
  audioElement = null;
374
+ mediaStream = null;
375
+ peerConnection = null;
376
+ audioSender = null;
268
377
  connectionWasActive = false;
378
+ connectionLossNotified = false;
379
+ isExplicitDisconnect = false;
380
+ isRecovering = false;
381
+ isCleaningUp = false;
269
382
  assistant = { text: "", started: false };
270
383
  latestHistory = [];
384
+ currentSessionConfig;
385
+ teardownTransportLifecycle;
386
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
387
+ this.emitConnectionLost("ice-disconnected");
388
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
271
389
  eventCallbacks = /* @__PURE__ */ new Set();
272
390
  pendingToolCalls = /* @__PURE__ */ new Map();
273
391
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -280,13 +398,23 @@ var OpenAIRealtimeAgentsClient = class {
280
398
  });
281
399
  async connect(config) {
282
400
  try {
401
+ if (this.isExplicitDisconnect && this.isRecovering) {
402
+ throw new Error("Realtime session disconnect requested");
403
+ }
283
404
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
405
+ this.isExplicitDisconnect = false;
406
+ this.isRecovering = false;
407
+ this.connectionLossNotified = false;
408
+ this.currentSessionConfig = config;
284
409
  this.audioElement = document.createElement("audio");
285
410
  this.audioElement.autoplay = true;
286
411
  this.audioElement.setAttribute("playsinline", "true");
412
+ this.mediaStream = await acquireMicrophoneStream();
413
+ this.bindTransportLifecycleEvents();
414
+ await this.lipSyncAnalyzer.prepareAudioContext();
287
415
  const agent = this.createAgent(config);
288
416
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
289
- this.createTransportOptions(this.audioElement)
417
+ this.createTransportOptions(this.audioElement, this.mediaStream)
290
418
  );
291
419
  this.session = new agentsRealtime.RealtimeSession(agent, {
292
420
  transport: this.transport,
@@ -318,6 +446,8 @@ var OpenAIRealtimeAgentsClient = class {
318
446
  }
319
447
  async disconnect() {
320
448
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
449
+ this.isExplicitDisconnect = true;
450
+ this.isRecovering = false;
321
451
  this.cleanup();
322
452
  }
323
453
  async updateSession(config) {
@@ -326,8 +456,27 @@ var OpenAIRealtimeAgentsClient = class {
326
456
  }
327
457
  this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
328
458
  await this.session.updateAgent(this.createAgent(config));
459
+ this.currentSessionConfig = config;
460
+ }
461
+ async recover(config) {
462
+ this.isRecovering = true;
463
+ this.connectionLossNotified = true;
464
+ this.currentSessionConfig = config;
465
+ try {
466
+ this.cleanupPendingToolCalls(
467
+ new Error("Realtime session interrupted during reconnect")
468
+ );
469
+ this.cleanup();
470
+ await this.connect(config ?? this.currentSessionConfig);
471
+ this.connectionLossNotified = false;
472
+ } finally {
473
+ this.isRecovering = false;
474
+ }
329
475
  }
330
476
  async sendText(text) {
477
+ if (this.isRecovering) {
478
+ throw new Error("Realtime transport reconnecting");
479
+ }
331
480
  if (!this.session) {
332
481
  throw new Error("Realtime session not active");
333
482
  }
@@ -335,11 +484,17 @@ var OpenAIRealtimeAgentsClient = class {
335
484
  this.session.sendMessage(text);
336
485
  }
337
486
  async sendAudio(_audio) {
487
+ if (this.isRecovering) {
488
+ throw new Error("Realtime transport reconnecting");
489
+ }
338
490
  console.warn(
339
491
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
340
492
  );
341
493
  }
342
494
  async sendToolResult(callId, output) {
495
+ if (this.isRecovering) {
496
+ throw new Error("Realtime transport reconnecting");
497
+ }
343
498
  const pendingCall = this.pendingToolCalls.get(callId);
344
499
  if (!pendingCall) {
345
500
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -348,6 +503,9 @@ var OpenAIRealtimeAgentsClient = class {
348
503
  pendingCall.resolve(output);
349
504
  }
350
505
  async interrupt() {
506
+ if (this.isRecovering) {
507
+ throw new Error("Realtime transport reconnecting");
508
+ }
351
509
  if (!this.session) {
352
510
  throw new Error("Realtime session not active");
353
511
  }
@@ -411,7 +569,7 @@ var OpenAIRealtimeAgentsClient = class {
411
569
  if (status === "disconnected" && this.connectionWasActive) {
412
570
  this.cleanupPendingToolCalls();
413
571
  this.connectionWasActive = false;
414
- this.emitEvent({ type: "session.ended" });
572
+ this.emitConnectionLost("connection-failed");
415
573
  }
416
574
  });
417
575
  }
@@ -513,10 +671,14 @@ var OpenAIRealtimeAgentsClient = class {
513
671
  });
514
672
  });
515
673
  }
516
- createTransportOptions(audioElement) {
674
+ createTransportOptions(audioElement, mediaStream) {
517
675
  return {
518
676
  audioElement,
677
+ mediaStream,
519
678
  changePeerConnection: async (peerConnection) => {
679
+ this.peerConnection = peerConnection;
680
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
681
+ this.bindPeerConnectionEvents(peerConnection);
520
682
  peerConnection.addEventListener("track", (event) => {
521
683
  const stream = event.streams[0];
522
684
  if (stream) {
@@ -527,7 +689,113 @@ var OpenAIRealtimeAgentsClient = class {
527
689
  }
528
690
  };
529
691
  }
692
+ bindPeerConnectionEvents(peerConnection) {
693
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
694
+ const iceState = peerConnection.iceConnectionState;
695
+ if (iceState === "failed") {
696
+ this.iceDisconnectDebouncer.cancel();
697
+ this.emitConnectionLost("ice-failed");
698
+ } else if (iceState === "disconnected") {
699
+ this.iceDisconnectDebouncer.schedule();
700
+ } else {
701
+ this.iceDisconnectDebouncer.cancel();
702
+ }
703
+ });
704
+ peerConnection.addEventListener("connectionstatechange", () => {
705
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
706
+ this.iceDisconnectDebouncer.cancel();
707
+ this.emitConnectionLost("connection-failed");
708
+ }
709
+ });
710
+ }
711
+ bindTransportLifecycleEvents() {
712
+ if (this.teardownTransportLifecycle) {
713
+ return;
714
+ }
715
+ this.teardownTransportLifecycle = bindTransportLifecycle({
716
+ onHidden: this.handleHidden,
717
+ onOnline: this.handleOnline,
718
+ onPageHide: this.handlePageHide,
719
+ onPageShow: this.handlePageShow,
720
+ onVisible: this.handleVisible,
721
+ onDeviceChange: this.handleDeviceChange
722
+ });
723
+ }
724
+ unbindTransportLifecycleEvents() {
725
+ this.teardownTransportLifecycle?.();
726
+ this.teardownTransportLifecycle = void 0;
727
+ }
728
+ handleOnline = () => {
729
+ this.emitConnectionLost("online");
730
+ };
731
+ handleHidden = () => {
732
+ this.lipSyncAnalyzer.pause();
733
+ };
734
+ handleVisible = () => {
735
+ this.lipSyncAnalyzer.resume();
736
+ this.emitConnectionLost("visibility");
737
+ };
738
+ handlePageHide = () => {
739
+ this.lipSyncAnalyzer.pause();
740
+ };
741
+ handlePageShow = (event) => {
742
+ this.lipSyncAnalyzer.resume();
743
+ if (event.persisted) {
744
+ this.emitConnectionLost("pageshow");
745
+ }
746
+ };
747
+ handleDeviceChange = () => {
748
+ void this.refreshMicrophoneTrack();
749
+ };
750
+ async refreshMicrophoneTrack() {
751
+ if (!this.peerConnection || this.isExplicitDisconnect) {
752
+ return;
753
+ }
754
+ try {
755
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
756
+ audioSender: this.audioSender,
757
+ mediaStream: this.mediaStream,
758
+ peerConnection: this.peerConnection
759
+ });
760
+ this.audioSender = audioSender;
761
+ this.mediaStream = mediaStream;
762
+ } catch (error) {
763
+ this.emitEvent({
764
+ type: "error",
765
+ error: error instanceof Error ? error : new Error(String(error))
766
+ });
767
+ }
768
+ }
769
+ emitConnectionLost(cause, error) {
770
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
771
+ return;
772
+ }
773
+ this.connectionLossNotified = true;
774
+ this.cleanupPendingToolCalls(
775
+ new Error("Realtime session interrupted during reconnect")
776
+ );
777
+ this.resetAssistantTracking();
778
+ this.lipSyncAnalyzer.stopOutput();
779
+ this.emitEvent({
780
+ type: "connection.lost",
781
+ cause,
782
+ error
783
+ });
784
+ }
785
+ shouldRecover(cause) {
786
+ if (!this.peerConnection) {
787
+ return false;
788
+ }
789
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
790
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
791
+ }
792
+ return true;
793
+ }
530
794
  cleanup(error) {
795
+ this.connectionWasActive = false;
796
+ this.isCleaningUp = true;
797
+ this.iceDisconnectDebouncer.cancel();
798
+ this.unbindTransportLifecycleEvents();
531
799
  this.cleanupPendingToolCalls(error);
532
800
  this.lipSyncAnalyzer.cleanup();
533
801
  if (this.transport) {
@@ -542,7 +810,15 @@ var OpenAIRealtimeAgentsClient = class {
542
810
  this.audioElement.srcObject = null;
543
811
  this.audioElement = null;
544
812
  }
813
+ if (this.mediaStream) {
814
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
815
+ this.mediaStream = null;
816
+ }
817
+ this.peerConnection = null;
818
+ this.audioSender = null;
545
819
  this.latestHistory = [];
820
+ this.connectionLossNotified = false;
821
+ this.isCleaningUp = false;
546
822
  this.resetAssistantTracking();
547
823
  }
548
824
  resetAssistantTracking() {