@charivo/realtime 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +32 -7
  2. package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/chunk-PDODY2T7.mjs +65 -0
  5. package/dist/chunk-PDODY2T7.mjs.map +1 -0
  6. package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
  7. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  8. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  9. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  10. package/dist/index.d.mts +16 -4
  11. package/dist/index.d.ts +16 -4
  12. package/dist/index.js +240 -38
  13. package/dist/index.js.map +1 -1
  14. package/dist/index.mjs +238 -98
  15. package/dist/index.mjs.map +1 -1
  16. package/dist/openai/index.d.mts +37 -1
  17. package/dist/openai/index.d.ts +37 -1
  18. package/dist/openai/index.js +427 -14
  19. package/dist/openai/index.js.map +1 -1
  20. package/dist/openai/index.mjs +3 -2
  21. package/dist/openai-agents/index.d.mts +26 -1
  22. package/dist/openai-agents/index.d.ts +26 -1
  23. package/dist/openai-agents/index.js +311 -18
  24. package/dist/openai-agents/index.js.map +1 -1
  25. package/dist/openai-agents/index.mjs +3 -2
  26. package/dist/remote/index.d.mts +3 -1
  27. package/dist/remote/index.d.ts +3 -1
  28. package/dist/remote/index.js +663 -32
  29. package/dist/remote/index.js.map +1 -1
  30. package/dist/remote/index.mjs +10 -3
  31. package/dist/remote/index.mjs.map +1 -1
  32. package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
  33. package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
  34. package/package.json +3 -3
  35. package/dist/chunk-IIKECRXN.mjs.map +0 -1
  36. package/dist/chunk-JGJZKKMH.mjs.map +0 -1
  37. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  38. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/remote/index.ts
7
7
 
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
85
+
8
86
  // src/internal/shared.ts
9
87
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
88
  function isRecord(value) {
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
83
161
  }
84
162
  audioAnalysisStream = null;
85
163
  audioContext = null;
164
+ audioSource = null;
86
165
  analyser = null;
87
166
  lipSyncInterval = null;
88
167
  pendingAudioElementPoll = null;
89
168
  observedAudioElement = null;
90
169
  observedAudioElementListener = null;
170
+ async prepareAudioContext() {
171
+ if (this.audioContext) {
172
+ return;
173
+ }
174
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
175
+ if (!audioContextConstructor) {
176
+ throw new Error("AudioContext is not supported in this browser");
177
+ }
178
+ this.audioContext = new audioContextConstructor();
179
+ }
91
180
  observeAudioElement(audioElement) {
92
181
  this.detachObservedAudioElement();
93
182
  const tryAttachStream = () => {
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
109
198
  this.stopLipSyncAnalysis();
110
199
  this.audioAnalysisStream = stream;
111
200
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
201
+ if (!this.audioContext) {
202
+ void this.prepareAudioContext().catch((error) => {
203
+ this.options.onError?.(error);
204
+ });
205
+ }
206
+ if (!this.audioContext) {
207
+ return;
115
208
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
209
+ this.audioSource?.disconnect?.();
210
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
211
  this.analyser = this.audioContext.createAnalyser();
119
212
  this.analyser.fftSize = 256;
120
213
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
214
+ this.audioSource.connect(this.analyser);
122
215
  this.startLipSyncAnalysis();
123
216
  } catch (error) {
124
217
  this.options.onError?.(error);
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
127
220
  stopOutput() {
128
221
  this.stopLipSyncAnalysis();
129
222
  }
223
+ pause() {
224
+ this.stopLipSyncAnalysis();
225
+ }
226
+ resume() {
227
+ if (!this.analyser || this.lipSyncInterval) {
228
+ return;
229
+ }
230
+ this.startLipSyncAnalysis();
231
+ }
130
232
  cleanup() {
131
233
  this.stopLipSyncAnalysis();
132
234
  this.stopAudioElementPolling();
133
235
  this.detachObservedAudioElement();
236
+ this.audioSource?.disconnect?.();
237
+ this.audioSource = null;
134
238
  if (this.audioContext) {
135
239
  void this.audioContext.close();
136
240
  this.audioContext = null;
@@ -183,18 +287,20 @@ var LipSyncAnalyzer = class {
183
287
  }
184
288
  };
185
289
 
290
+ // src/instructions.ts
291
+ var DEFAULT_REALTIME_VOICE = "marin";
292
+
186
293
  // src/openai-agents/session-config.ts
187
294
  var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
188
295
  var DEFAULT_MODEL = "gpt-realtime-mini";
189
- var DEFAULT_VOICE = "marin";
190
296
  function resolveInstructions(config) {
191
297
  return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
192
298
  }
193
299
  function resolveVoice(config) {
194
- return config?.voice ?? DEFAULT_VOICE;
300
+ return config?.voice ?? DEFAULT_REALTIME_VOICE;
195
301
  }
196
302
  function toOpenAIRealtimeAgentsSessionConfig(config) {
197
- return {
303
+ const sessionConfig = {
198
304
  model: config?.model ?? DEFAULT_MODEL,
199
305
  instructions: resolveInstructions(config),
200
306
  toolChoice: config?.toolChoice ?? "auto",
@@ -205,6 +311,13 @@ function toOpenAIRealtimeAgentsSessionConfig(config) {
205
311
  }
206
312
  }
207
313
  };
314
+ if (config?.temperature !== void 0) {
315
+ sessionConfig.temperature = config.temperature;
316
+ }
317
+ if (config?.maxTokens !== void 0) {
318
+ sessionConfig.maxResponseOutputTokens = config.maxTokens;
319
+ }
320
+ return sessionConfig;
208
321
  }
209
322
 
210
323
  // src/openai-agents/tool-schema.ts
@@ -256,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
256
369
  session = null;
257
370
  transport = null;
258
371
  audioElement = null;
372
+ mediaStream = null;
373
+ peerConnection = null;
374
+ audioSender = null;
259
375
  connectionWasActive = false;
376
+ connectionLossNotified = false;
377
+ isExplicitDisconnect = false;
378
+ isRecovering = false;
379
+ isCleaningUp = false;
260
380
  assistant = { text: "", started: false };
261
381
  latestHistory = [];
382
+ currentSessionConfig;
383
+ teardownTransportLifecycle;
384
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
385
+ this.emitConnectionLost("ice-disconnected");
386
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
262
387
  eventCallbacks = /* @__PURE__ */ new Set();
263
388
  pendingToolCalls = /* @__PURE__ */ new Map();
264
389
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -271,18 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
271
396
  });
272
397
  async connect(config) {
273
398
  try {
399
+ if (this.isExplicitDisconnect && this.isRecovering) {
400
+ throw new Error("Realtime session disconnect requested");
401
+ }
274
402
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
403
+ this.isExplicitDisconnect = false;
404
+ this.isRecovering = false;
405
+ this.connectionLossNotified = false;
406
+ this.currentSessionConfig = config;
275
407
  this.audioElement = document.createElement("audio");
276
408
  this.audioElement.autoplay = true;
277
409
  this.audioElement.setAttribute("playsinline", "true");
278
- const agent = new agentsRealtime.RealtimeAgent({
279
- name: "charivo-realtime-agent",
280
- instructions: resolveInstructions(config),
281
- tools: this.createProxyTools(config?.tools),
282
- voice: resolveVoice(config)
283
- });
410
+ this.mediaStream = await acquireMicrophoneStream();
411
+ this.bindTransportLifecycleEvents();
412
+ await this.lipSyncAnalyzer.prepareAudioContext();
413
+ const agent = this.createAgent(config);
284
414
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
285
- this.createTransportOptions(this.audioElement)
415
+ this.createTransportOptions(this.audioElement, this.mediaStream)
286
416
  );
287
417
  this.session = new agentsRealtime.RealtimeSession(agent, {
288
418
  transport: this.transport,
@@ -314,9 +444,37 @@ var OpenAIRealtimeAgentsClient = class {
314
444
  }
315
445
  async disconnect() {
316
446
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
447
+ this.isExplicitDisconnect = true;
448
+ this.isRecovering = false;
317
449
  this.cleanup();
318
450
  }
451
+ async updateSession(config) {
452
+ if (!this.session) {
453
+ throw new Error("Realtime session not active");
454
+ }
455
+ this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
456
+ await this.session.updateAgent(this.createAgent(config));
457
+ this.currentSessionConfig = config;
458
+ }
459
+ async recover(config) {
460
+ this.isRecovering = true;
461
+ this.connectionLossNotified = true;
462
+ this.currentSessionConfig = config;
463
+ try {
464
+ this.cleanupPendingToolCalls(
465
+ new Error("Realtime session interrupted during reconnect")
466
+ );
467
+ this.cleanup();
468
+ await this.connect(config ?? this.currentSessionConfig);
469
+ this.connectionLossNotified = false;
470
+ } finally {
471
+ this.isRecovering = false;
472
+ }
473
+ }
319
474
  async sendText(text) {
475
+ if (this.isRecovering) {
476
+ throw new Error("Realtime transport reconnecting");
477
+ }
320
478
  if (!this.session) {
321
479
  throw new Error("Realtime session not active");
322
480
  }
@@ -324,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
324
482
  this.session.sendMessage(text);
325
483
  }
326
484
  async sendAudio(_audio) {
485
+ if (this.isRecovering) {
486
+ throw new Error("Realtime transport reconnecting");
487
+ }
327
488
  console.warn(
328
489
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
329
490
  );
330
491
  }
331
492
  async sendToolResult(callId, output) {
493
+ if (this.isRecovering) {
494
+ throw new Error("Realtime transport reconnecting");
495
+ }
332
496
  const pendingCall = this.pendingToolCalls.get(callId);
333
497
  if (!pendingCall) {
334
498
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -337,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
337
501
  pendingCall.resolve(output);
338
502
  }
339
503
  async interrupt() {
504
+ if (this.isRecovering) {
505
+ throw new Error("Realtime transport reconnecting");
506
+ }
340
507
  if (!this.session) {
341
508
  throw new Error("Realtime session not active");
342
509
  }
@@ -346,6 +513,14 @@ var OpenAIRealtimeAgentsClient = class {
346
513
  onEvent(callback) {
347
514
  this.eventCallbacks.add(callback);
348
515
  }
516
+ createAgent(config) {
517
+ return new agentsRealtime.RealtimeAgent({
518
+ name: "charivo-realtime-agent",
519
+ instructions: resolveInstructions(config),
520
+ tools: this.createProxyTools(config?.tools),
521
+ voice: resolveVoice(config)
522
+ });
523
+ }
349
524
  bindSessionEvents(session) {
350
525
  session.on("agent_start", () => {
351
526
  this.ensureAssistantStarted();
@@ -392,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
392
567
  if (status === "disconnected" && this.connectionWasActive) {
393
568
  this.cleanupPendingToolCalls();
394
569
  this.connectionWasActive = false;
395
- this.emitEvent({ type: "session.ended" });
570
+ this.emitConnectionLost("connection-failed");
396
571
  }
397
572
  });
398
573
  }
@@ -494,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
494
669
  });
495
670
  });
496
671
  }
497
- createTransportOptions(audioElement) {
672
+ createTransportOptions(audioElement, mediaStream) {
498
673
  return {
499
674
  audioElement,
675
+ mediaStream,
500
676
  changePeerConnection: async (peerConnection) => {
677
+ this.peerConnection = peerConnection;
678
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
679
+ this.bindPeerConnectionEvents(peerConnection);
501
680
  peerConnection.addEventListener("track", (event) => {
502
681
  const stream = event.streams[0];
503
682
  if (stream) {
@@ -508,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
508
687
  }
509
688
  };
510
689
  }
690
+ bindPeerConnectionEvents(peerConnection) {
691
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
692
+ const iceState = peerConnection.iceConnectionState;
693
+ if (iceState === "failed") {
694
+ this.iceDisconnectDebouncer.cancel();
695
+ this.emitConnectionLost("ice-failed");
696
+ } else if (iceState === "disconnected") {
697
+ this.iceDisconnectDebouncer.schedule();
698
+ } else {
699
+ this.iceDisconnectDebouncer.cancel();
700
+ }
701
+ });
702
+ peerConnection.addEventListener("connectionstatechange", () => {
703
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
704
+ this.iceDisconnectDebouncer.cancel();
705
+ this.emitConnectionLost("connection-failed");
706
+ }
707
+ });
708
+ }
709
+ bindTransportLifecycleEvents() {
710
+ if (this.teardownTransportLifecycle) {
711
+ return;
712
+ }
713
+ this.teardownTransportLifecycle = bindTransportLifecycle({
714
+ onHidden: this.handleHidden,
715
+ onOnline: this.handleOnline,
716
+ onPageHide: this.handlePageHide,
717
+ onPageShow: this.handlePageShow,
718
+ onVisible: this.handleVisible,
719
+ onDeviceChange: this.handleDeviceChange
720
+ });
721
+ }
722
+ unbindTransportLifecycleEvents() {
723
+ this.teardownTransportLifecycle?.();
724
+ this.teardownTransportLifecycle = void 0;
725
+ }
726
+ handleOnline = () => {
727
+ this.emitConnectionLost("online");
728
+ };
729
+ handleHidden = () => {
730
+ this.lipSyncAnalyzer.pause();
731
+ };
732
+ handleVisible = () => {
733
+ this.lipSyncAnalyzer.resume();
734
+ this.emitConnectionLost("visibility");
735
+ };
736
+ handlePageHide = () => {
737
+ this.lipSyncAnalyzer.pause();
738
+ };
739
+ handlePageShow = (event) => {
740
+ this.lipSyncAnalyzer.resume();
741
+ if (event.persisted) {
742
+ this.emitConnectionLost("pageshow");
743
+ }
744
+ };
745
+ handleDeviceChange = () => {
746
+ void this.refreshMicrophoneTrack();
747
+ };
748
+ async refreshMicrophoneTrack() {
749
+ if (!this.peerConnection || this.isExplicitDisconnect) {
750
+ return;
751
+ }
752
+ try {
753
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
754
+ audioSender: this.audioSender,
755
+ mediaStream: this.mediaStream,
756
+ peerConnection: this.peerConnection
757
+ });
758
+ this.audioSender = audioSender;
759
+ this.mediaStream = mediaStream;
760
+ } catch (error) {
761
+ this.emitEvent({
762
+ type: "error",
763
+ error: error instanceof Error ? error : new Error(String(error))
764
+ });
765
+ }
766
+ }
767
+ emitConnectionLost(cause, error) {
768
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
769
+ return;
770
+ }
771
+ this.connectionLossNotified = true;
772
+ this.cleanupPendingToolCalls(
773
+ new Error("Realtime session interrupted during reconnect")
774
+ );
775
+ this.resetAssistantTracking();
776
+ this.lipSyncAnalyzer.stopOutput();
777
+ this.emitEvent({
778
+ type: "connection.lost",
779
+ cause,
780
+ error
781
+ });
782
+ }
783
+ shouldRecover(cause) {
784
+ if (!this.peerConnection) {
785
+ return false;
786
+ }
787
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
788
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
789
+ }
790
+ return true;
791
+ }
511
792
  cleanup(error) {
793
+ this.connectionWasActive = false;
794
+ this.isCleaningUp = true;
795
+ this.iceDisconnectDebouncer.cancel();
796
+ this.unbindTransportLifecycleEvents();
512
797
  this.cleanupPendingToolCalls(error);
513
798
  this.lipSyncAnalyzer.cleanup();
514
799
  if (this.transport) {
@@ -523,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
523
808
  this.audioElement.srcObject = null;
524
809
  this.audioElement = null;
525
810
  }
811
+ if (this.mediaStream) {
812
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
813
+ this.mediaStream = null;
814
+ }
815
+ this.peerConnection = null;
816
+ this.audioSender = null;
526
817
  this.latestHistory = [];
818
+ this.connectionLossNotified = false;
819
+ this.isCleaningUp = false;
527
820
  this.resetAssistantTracking();
528
821
  }
529
822
  resetAssistantTracking() {
@@ -569,6 +862,7 @@ var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
569
862
  "conversation.item.input_audio_transcription.completed",
570
863
  "error"
571
864
  ]);
865
+ var DEFAULT_SESSION_UPDATE_TIMEOUT_MS = 5e3;
572
866
  var OpenAIRealtimeClient = class {
573
867
  constructor(options = {}) {
574
868
  this.options = options;
@@ -577,7 +871,9 @@ var OpenAIRealtimeClient = class {
577
871
  dc = null;
578
872
  audioElement = null;
579
873
  mediaStream = null;
874
+ audioSender = null;
580
875
  audioContext = null;
876
+ audioSource = null;
581
877
  analyser = null;
582
878
  lipSyncInterval = null;
583
879
  isResponseInProgress = false;
@@ -585,13 +881,35 @@ var OpenAIRealtimeClient = class {
585
881
  hasStartedAssistantResponse = false;
586
882
  hasStartedAudioOutput = false;
587
883
  assistantText = "";
884
+ nextSessionUpdateId = 1;
885
+ pendingSessionUpdate = null;
886
+ currentSessionConfig;
887
+ connectionLossNotified = false;
888
+ isExplicitDisconnect = false;
889
+ isRecovering = false;
890
+ isCleaningUp = false;
891
+ teardownTransportLifecycle;
892
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
893
+ this.emitConnectionLost("ice-disconnected");
894
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
588
895
  eventCallbacks = /* @__PURE__ */ new Set();
589
896
  async connect(config) {
590
897
  try {
898
+ if (this.isExplicitDisconnect && this.isRecovering) {
899
+ throw new Error("Realtime session disconnect requested");
900
+ }
591
901
  this.log("Starting OpenAI Realtime WebRTC connection");
902
+ this.isExplicitDisconnect = false;
903
+ this.isRecovering = false;
904
+ this.connectionLossNotified = false;
905
+ this.currentSessionConfig = config;
592
906
  this.pc = new RTCPeerConnection();
593
907
  this.audioElement = document.createElement("audio");
594
908
  this.audioElement.autoplay = true;
909
+ this.audioElement.setAttribute("playsinline", "true");
910
+ this.prepareAudioAnalysis();
911
+ this.bindConnectionEvents();
912
+ this.bindTransportLifecycleEvents();
595
913
  this.pc.ontrack = (event) => {
596
914
  if (this.audioElement) {
597
915
  this.audioElement.srcObject = event.streams[0];
@@ -599,11 +917,9 @@ var OpenAIRealtimeClient = class {
599
917
  this.setupAudioAnalysis(event.streams[0]);
600
918
  };
601
919
  try {
602
- this.mediaStream = await navigator.mediaDevices.getUserMedia({
603
- audio: true
604
- });
920
+ this.mediaStream = await acquireMicrophoneStream();
605
921
  const audioTrack = this.mediaStream.getTracks()[0];
606
- this.pc.addTrack(audioTrack, this.mediaStream);
922
+ this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
607
923
  } catch {
608
924
  throw new Error("Microphone access required for Realtime API");
609
925
  }
@@ -642,9 +958,64 @@ var OpenAIRealtimeClient = class {
642
958
  }
643
959
  async disconnect() {
644
960
  this.log("Disconnecting OpenAI Realtime WebRTC");
961
+ this.isExplicitDisconnect = true;
962
+ this.isRecovering = false;
645
963
  this.cleanup();
646
964
  }
965
+ async updateSession(config) {
966
+ if (this.isResponseInProgress) {
967
+ throw new Error(
968
+ "Cannot update the realtime session while a response is in progress. Call interrupt() first."
969
+ );
970
+ }
971
+ const dc = this.requireOpenDataChannel();
972
+ const eventId = `charivo-session-update-${this.nextSessionUpdateId++}`;
973
+ if (this.pendingSessionUpdate) {
974
+ throw new Error("Realtime session update already in progress");
975
+ }
976
+ dc.send(
977
+ JSON.stringify({
978
+ type: "session.update",
979
+ event_id: eventId,
980
+ session: toOpenAIRealtimeSessionUpdate(config)
981
+ })
982
+ );
983
+ await new Promise((resolve, reject) => {
984
+ const timeoutId = setTimeout(() => {
985
+ this.rejectPendingSessionUpdate(
986
+ new Error(
987
+ `Timed out waiting for session.updated after ${this.getSessionUpdateTimeoutMs()}ms`
988
+ )
989
+ );
990
+ }, this.getSessionUpdateTimeoutMs());
991
+ this.pendingSessionUpdate = {
992
+ eventId,
993
+ resolve,
994
+ reject,
995
+ timeoutId
996
+ };
997
+ });
998
+ this.currentSessionConfig = config;
999
+ }
1000
+ async recover(config) {
1001
+ this.isRecovering = true;
1002
+ this.connectionLossNotified = true;
1003
+ this.currentSessionConfig = config;
1004
+ try {
1005
+ if (await this.tryIceRestartRecovery(config)) {
1006
+ this.connectionLossNotified = false;
1007
+ return;
1008
+ }
1009
+ await this.rebuildConnection(config);
1010
+ this.connectionLossNotified = false;
1011
+ } finally {
1012
+ this.isRecovering = false;
1013
+ }
1014
+ }
647
1015
  async sendText(text) {
1016
+ if (this.isRecovering) {
1017
+ throw new Error("Realtime transport reconnecting");
1018
+ }
648
1019
  if (!this.dc || this.dc.readyState !== "open") {
649
1020
  throw new Error("DataChannel not ready");
650
1021
  }
@@ -671,11 +1042,17 @@ var OpenAIRealtimeClient = class {
671
1042
  this.beginResponseRequest();
672
1043
  }
673
1044
  async sendAudio(_audio) {
1045
+ if (this.isRecovering) {
1046
+ throw new Error("Realtime transport reconnecting");
1047
+ }
674
1048
  console.warn(
675
1049
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
676
1050
  );
677
1051
  }
678
1052
  async sendToolResult(callId, output) {
1053
+ if (this.isRecovering) {
1054
+ throw new Error("Realtime transport reconnecting");
1055
+ }
679
1056
  const dc = this.requireOpenDataChannel();
680
1057
  dc.send(
681
1058
  JSON.stringify({
@@ -691,6 +1068,9 @@ var OpenAIRealtimeClient = class {
691
1068
  this.beginResponseRequest();
692
1069
  }
693
1070
  async interrupt() {
1071
+ if (this.isRecovering) {
1072
+ throw new Error("Realtime transport reconnecting");
1073
+ }
694
1074
  const dc = this.requireOpenDataChannel();
695
1075
  if (!this.isResponseInProgress) {
696
1076
  return;
@@ -708,7 +1088,9 @@ var OpenAIRealtimeClient = class {
708
1088
  }
709
1089
  switch (event.type) {
710
1090
  case "session.created":
1091
+ return;
711
1092
  case "session.updated":
1093
+ this.resolvePendingSessionUpdate();
712
1094
  return;
713
1095
  case "response.audio.delta":
714
1096
  case "output_audio_buffer.started":
@@ -814,10 +1196,16 @@ var OpenAIRealtimeClient = class {
814
1196
  case "error":
815
1197
  this.isResponseInProgress = false;
816
1198
  this.cancelInFlight = false;
817
- this.emitEvent({
818
- type: "error",
819
- error: new Error(event.error?.message || "Unknown error")
820
- });
1199
+ {
1200
+ const error = new Error(event.error?.message || "Unknown error");
1201
+ if (this.isPendingSessionUpdateError(event)) {
1202
+ this.rejectPendingSessionUpdate(error);
1203
+ }
1204
+ this.emitEvent({
1205
+ type: "error",
1206
+ error
1207
+ });
1208
+ }
821
1209
  return;
822
1210
  default:
823
1211
  return;
@@ -879,18 +1267,133 @@ var OpenAIRealtimeClient = class {
879
1267
  }
880
1268
  return bootstrap;
881
1269
  }
1270
+ bindConnectionEvents() {
1271
+ if (!this.pc) {
1272
+ return;
1273
+ }
1274
+ this.pc.addEventListener("iceconnectionstatechange", () => {
1275
+ const iceState = this.pc?.iceConnectionState;
1276
+ if (iceState === "failed") {
1277
+ this.iceDisconnectDebouncer.cancel();
1278
+ this.emitConnectionLost("ice-failed");
1279
+ } else if (iceState === "disconnected") {
1280
+ this.iceDisconnectDebouncer.schedule();
1281
+ } else {
1282
+ this.iceDisconnectDebouncer.cancel();
1283
+ }
1284
+ });
1285
+ this.pc.addEventListener("connectionstatechange", () => {
1286
+ const connectionState = this.pc?.connectionState;
1287
+ if (connectionState === "failed" || connectionState === "closed") {
1288
+ this.iceDisconnectDebouncer.cancel();
1289
+ this.emitConnectionLost("connection-failed");
1290
+ }
1291
+ });
1292
+ }
1293
+ bindTransportLifecycleEvents() {
1294
+ if (this.teardownTransportLifecycle) {
1295
+ return;
1296
+ }
1297
+ this.teardownTransportLifecycle = bindTransportLifecycle({
1298
+ onHidden: this.handleHidden,
1299
+ onOnline: this.handleOnline,
1300
+ onPageHide: this.handlePageHide,
1301
+ onPageShow: this.handlePageShow,
1302
+ onVisible: this.handleVisible,
1303
+ onDeviceChange: this.handleDeviceChange
1304
+ });
1305
+ }
1306
+ unbindTransportLifecycleEvents() {
1307
+ this.teardownTransportLifecycle?.();
1308
+ this.teardownTransportLifecycle = void 0;
1309
+ }
1310
+ handleOnline = () => {
1311
+ this.emitConnectionLost("online");
1312
+ };
1313
+ handleHidden = () => {
1314
+ this.pauseAudioAnalysis();
1315
+ };
1316
+ handleVisible = () => {
1317
+ this.resumeAudioAnalysis();
1318
+ this.emitConnectionLost("visibility");
1319
+ };
1320
+ handlePageHide = () => {
1321
+ this.pauseAudioAnalysis();
1322
+ };
1323
+ handlePageShow = (event) => {
1324
+ this.resumeAudioAnalysis();
1325
+ if (event.persisted) {
1326
+ this.emitConnectionLost("pageshow");
1327
+ }
1328
+ };
1329
+ handleDeviceChange = () => {
1330
+ void this.refreshMicrophoneTrack();
1331
+ };
1332
+ async refreshMicrophoneTrack() {
1333
+ if (!this.pc || this.isExplicitDisconnect) {
1334
+ return;
1335
+ }
1336
+ try {
1337
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
1338
+ audioSender: this.audioSender,
1339
+ mediaStream: this.mediaStream,
1340
+ peerConnection: this.pc
1341
+ });
1342
+ this.audioSender = audioSender;
1343
+ this.mediaStream = mediaStream;
1344
+ } catch (error) {
1345
+ this.emitEvent({
1346
+ type: "error",
1347
+ error: error instanceof Error ? error : new Error(String(error))
1348
+ });
1349
+ }
1350
+ }
1351
+ emitConnectionLost(cause, error) {
1352
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
1353
+ return;
1354
+ }
1355
+ this.connectionLossNotified = true;
1356
+ this.isResponseInProgress = false;
1357
+ this.cancelInFlight = false;
1358
+ this.resetResponseTracking();
1359
+ this.stopLipSyncAnalysis();
1360
+ this.emitEvent({
1361
+ type: "connection.lost",
1362
+ cause,
1363
+ error
1364
+ });
1365
+ }
1366
+ needsRecovery(cause) {
1367
+ if (!this.pc || !this.dc) {
1368
+ return false;
1369
+ }
1370
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
1371
+ return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
1372
+ }
1373
+ return true;
1374
+ }
1375
+ prepareAudioAnalysis() {
1376
+ if (this.audioContext) {
1377
+ return;
1378
+ }
1379
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
1380
+ if (!audioContextConstructor) {
1381
+ return;
1382
+ }
1383
+ this.audioContext = new audioContextConstructor();
1384
+ }
882
1385
  setupAudioAnalysis(stream) {
883
1386
  try {
884
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
885
- if (!audioContextConstructor) {
886
- throw new Error("AudioContext is not supported in this browser");
1387
+ this.prepareAudioAnalysis();
1388
+ if (!this.audioContext) {
1389
+ return;
887
1390
  }
888
- this.audioContext = new audioContextConstructor();
889
- const source = this.audioContext.createMediaStreamSource(stream);
1391
+ this.audioSource?.disconnect();
1392
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
890
1393
  this.analyser = this.audioContext.createAnalyser();
891
1394
  this.analyser.fftSize = 256;
892
1395
  this.analyser.smoothingTimeConstant = 0.8;
893
- source.connect(this.analyser);
1396
+ this.audioSource.connect(this.analyser);
894
1397
  this.startLipSyncAnalysis();
895
1398
  } catch (error) {
896
1399
  console.error("Failed to setup audio analysis:", error);
@@ -926,8 +1429,23 @@ var OpenAIRealtimeClient = class {
926
1429
  }
927
1430
  this.emitEvent({ type: "audio.lipsync", rms: 0 });
928
1431
  }
1432
+ pauseAudioAnalysis() {
1433
+ this.stopLipSyncAnalysis();
1434
+ }
1435
+ resumeAudioAnalysis() {
1436
+ if (!this.analyser || this.lipSyncInterval) {
1437
+ return;
1438
+ }
1439
+ this.startLipSyncAnalysis();
1440
+ }
929
1441
  cleanup() {
1442
+ this.isCleaningUp = true;
1443
+ this.unbindTransportLifecycleEvents();
1444
+ this.iceDisconnectDebouncer.cancel();
930
1445
  this.stopLipSyncAnalysis();
1446
+ this.rejectPendingSessionUpdate(
1447
+ new Error("Realtime session ended before session update completed")
1448
+ );
931
1449
  if (this.dc) {
932
1450
  this.dc.close();
933
1451
  this.dc = null;
@@ -948,9 +1466,13 @@ var OpenAIRealtimeClient = class {
948
1466
  this.audioContext.close();
949
1467
  this.audioContext = null;
950
1468
  }
1469
+ this.audioSource = null;
1470
+ this.audioSender = null;
951
1471
  this.analyser = null;
952
1472
  this.isResponseInProgress = false;
953
1473
  this.cancelInFlight = false;
1474
+ this.connectionLossNotified = false;
1475
+ this.isCleaningUp = false;
954
1476
  this.resetResponseTracking();
955
1477
  }
956
1478
  beginResponseRequest() {
@@ -959,11 +1481,86 @@ var OpenAIRealtimeClient = class {
959
1481
  this.cancelInFlight = false;
960
1482
  }
961
1483
  requireOpenDataChannel() {
1484
+ if (this.isRecovering) {
1485
+ throw new Error("Realtime transport reconnecting");
1486
+ }
962
1487
  if (!this.dc || this.dc.readyState !== "open") {
963
1488
  throw new Error("DataChannel not ready");
964
1489
  }
965
1490
  return this.dc;
966
1491
  }
1492
+ async tryIceRestartRecovery(config) {
1493
+ const pc = this.pc;
1494
+ if (!pc || this.shouldRebuildConnection()) {
1495
+ return false;
1496
+ }
1497
+ if (pc.iceConnectionState === "connected") {
1498
+ return true;
1499
+ }
1500
+ if (pc.iceConnectionState === "disconnected") {
1501
+ await delay(1500);
1502
+ if (pc.iceConnectionState !== "disconnected") {
1503
+ return true;
1504
+ }
1505
+ }
1506
+ if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
1507
+ return false;
1508
+ }
1509
+ if (typeof pc.restartIce === "function") {
1510
+ pc.restartIce();
1511
+ }
1512
+ const offer = await pc.createOffer({ iceRestart: true });
1513
+ await pc.setLocalDescription(offer);
1514
+ const bootstrap = await this.getSessionBootstrap({
1515
+ transport: "webrtc",
1516
+ session: config ?? this.currentSessionConfig ?? {},
1517
+ sdpOffer: offer.sdp
1518
+ });
1519
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1520
+ throw new Error(
1521
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1522
+ );
1523
+ }
1524
+ await pc.setRemoteDescription({
1525
+ type: "answer",
1526
+ sdp: bootstrap.answerSdp
1527
+ });
1528
+ return true;
1529
+ }
1530
+ shouldRebuildConnection() {
1531
+ return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
1532
+ }
1533
+ async rebuildConnection(config) {
1534
+ this.connectionLossNotified = true;
1535
+ this.cleanup();
1536
+ await this.connect(config ?? this.currentSessionConfig);
1537
+ }
1538
+ resolvePendingSessionUpdate() {
1539
+ if (!this.pendingSessionUpdate) {
1540
+ return;
1541
+ }
1542
+ clearTimeout(this.pendingSessionUpdate.timeoutId);
1543
+ this.pendingSessionUpdate.resolve();
1544
+ this.pendingSessionUpdate = null;
1545
+ }
1546
+ rejectPendingSessionUpdate(error) {
1547
+ if (!this.pendingSessionUpdate) {
1548
+ return;
1549
+ }
1550
+ clearTimeout(this.pendingSessionUpdate.timeoutId);
1551
+ this.pendingSessionUpdate.reject(error);
1552
+ this.pendingSessionUpdate = null;
1553
+ }
1554
+ isPendingSessionUpdateError(event) {
1555
+ const pendingUpdate = this.pendingSessionUpdate;
1556
+ if (!pendingUpdate) {
1557
+ return false;
1558
+ }
1559
+ return event.error?.event_id === pendingUpdate.eventId;
1560
+ }
1561
+ getSessionUpdateTimeoutMs() {
1562
+ return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
1563
+ }
967
1564
  resetResponseTracking() {
968
1565
  this.assistantText = "";
969
1566
  this.hasStartedAssistantResponse = false;
@@ -989,6 +1586,34 @@ var OpenAIRealtimeClient = class {
989
1586
  function createOpenAIRealtimeClient(options) {
990
1587
  return new OpenAIRealtimeClient(options);
991
1588
  }
1589
+ function toOpenAIRealtimeSessionUpdate(config) {
1590
+ const session = {
1591
+ audio: {
1592
+ output: {
1593
+ voice: config?.voice ?? DEFAULT_REALTIME_VOICE
1594
+ }
1595
+ },
1596
+ tool_choice: config?.toolChoice ?? "auto"
1597
+ };
1598
+ if (config?.instructions) {
1599
+ session.instructions = config.instructions;
1600
+ }
1601
+ if (config?.temperature !== void 0) {
1602
+ session.temperature = config.temperature;
1603
+ }
1604
+ if (config?.maxTokens !== void 0) {
1605
+ session.max_response_output_tokens = config.maxTokens;
1606
+ }
1607
+ if (config?.tools?.length) {
1608
+ session.tools = config.tools;
1609
+ }
1610
+ return session;
1611
+ }
1612
+ function delay(ms) {
1613
+ return new Promise((resolve) => {
1614
+ setTimeout(resolve, ms);
1615
+ });
1616
+ }
992
1617
 
993
1618
  // src/remote/index.ts
994
1619
  var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
@@ -1037,6 +1662,12 @@ var RemoteRealtimeClient = class {
1037
1662
  throw error;
1038
1663
  }
1039
1664
  }
1665
+ async updateSession(config) {
1666
+ await this.getActiveTransportClient().updateSession(config);
1667
+ }
1668
+ async recover(config) {
1669
+ await this.getActiveTransportClient().recover(config);
1670
+ }
1040
1671
  async disconnect() {
1041
1672
  if (!this.transportClient) {
1042
1673
  return;