@charivo/realtime 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +32 -7
  2. package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/chunk-PDODY2T7.mjs +65 -0
  5. package/dist/chunk-PDODY2T7.mjs.map +1 -0
  6. package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
  7. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  8. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  9. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  10. package/dist/index.d.mts +16 -4
  11. package/dist/index.d.ts +16 -4
  12. package/dist/index.js +240 -38
  13. package/dist/index.js.map +1 -1
  14. package/dist/index.mjs +238 -98
  15. package/dist/index.mjs.map +1 -1
  16. package/dist/openai/index.d.mts +37 -1
  17. package/dist/openai/index.d.ts +37 -1
  18. package/dist/openai/index.js +427 -14
  19. package/dist/openai/index.js.map +1 -1
  20. package/dist/openai/index.mjs +3 -2
  21. package/dist/openai-agents/index.d.mts +26 -1
  22. package/dist/openai-agents/index.d.ts +26 -1
  23. package/dist/openai-agents/index.js +311 -18
  24. package/dist/openai-agents/index.js.map +1 -1
  25. package/dist/openai-agents/index.mjs +3 -2
  26. package/dist/remote/index.d.mts +3 -1
  27. package/dist/remote/index.d.ts +3 -1
  28. package/dist/remote/index.js +663 -32
  29. package/dist/remote/index.js.map +1 -1
  30. package/dist/remote/index.mjs +10 -3
  31. package/dist/remote/index.mjs.map +1 -1
  32. package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
  33. package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
  34. package/package.json +3 -3
  35. package/dist/chunk-IIKECRXN.mjs.map +0 -1
  36. package/dist/chunk-JGJZKKMH.mjs.map +0 -1
  37. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  38. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
@@ -5,6 +5,84 @@ var agentsRealtime = require('@openai/agents-realtime');
5
5
 
6
6
  // src/openai-agents/client.ts
7
7
 
8
+ // src/internal/microphone.ts
9
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
10
+ echoCancellation: true,
11
+ noiseSuppression: true,
12
+ autoGainControl: true
13
+ };
14
+ async function acquireMicrophoneStream() {
15
+ return navigator.mediaDevices.getUserMedia({
16
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
17
+ });
18
+ }
19
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
20
+ function bindTransportLifecycle(callbacks) {
21
+ const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
22
+ onHidden: callbacks.onHidden,
23
+ onOnline: callbacks.onOnline,
24
+ onPageHide: callbacks.onPageHide,
25
+ onPageShow: callbacks.onPageShow,
26
+ onVisible: callbacks.onVisible
27
+ });
28
+ if (callbacks.onDeviceChange) {
29
+ navigator.mediaDevices?.addEventListener?.(
30
+ "devicechange",
31
+ callbacks.onDeviceChange
32
+ );
33
+ }
34
+ return () => {
35
+ teardownBrowserLifecycle();
36
+ if (callbacks.onDeviceChange) {
37
+ navigator.mediaDevices?.removeEventListener?.(
38
+ "devicechange",
39
+ callbacks.onDeviceChange
40
+ );
41
+ }
42
+ };
43
+ }
44
+ async function replaceMicrophoneTrack(state) {
45
+ if (!state.peerConnection) {
46
+ throw new Error("Realtime transport not active");
47
+ }
48
+ const nextStream = await acquireMicrophoneStream();
49
+ const nextTrack = nextStream.getAudioTracks()[0];
50
+ if (!nextTrack) {
51
+ throw new Error("Microphone access required for Realtime API");
52
+ }
53
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
54
+ if (!sender) {
55
+ throw new Error("No outbound audio sender available");
56
+ }
57
+ await sender.replaceTrack(nextTrack);
58
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
59
+ return {
60
+ audioSender: sender,
61
+ mediaStream: nextStream
62
+ };
63
+ }
64
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
65
+ let pending = null;
66
+ return {
67
+ schedule() {
68
+ if (pending) {
69
+ return;
70
+ }
71
+ pending = setTimeout(() => {
72
+ pending = null;
73
+ onDisconnect();
74
+ }, delayMs);
75
+ },
76
+ cancel() {
77
+ if (!pending) {
78
+ return;
79
+ }
80
+ clearTimeout(pending);
81
+ pending = null;
82
+ }
83
+ };
84
+ }
85
+
8
86
  // src/internal/shared.ts
9
87
  var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
88
  function isRecord(value) {
@@ -83,11 +161,22 @@ var LipSyncAnalyzer = class {
83
161
  }
84
162
  audioAnalysisStream = null;
85
163
  audioContext = null;
164
+ audioSource = null;
86
165
  analyser = null;
87
166
  lipSyncInterval = null;
88
167
  pendingAudioElementPoll = null;
89
168
  observedAudioElement = null;
90
169
  observedAudioElementListener = null;
170
+ async prepareAudioContext() {
171
+ if (this.audioContext) {
172
+ return;
173
+ }
174
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
175
+ if (!audioContextConstructor) {
176
+ throw new Error("AudioContext is not supported in this browser");
177
+ }
178
+ this.audioContext = new audioContextConstructor();
179
+ }
91
180
  observeAudioElement(audioElement) {
92
181
  this.detachObservedAudioElement();
93
182
  const tryAttachStream = () => {
@@ -109,16 +198,20 @@ var LipSyncAnalyzer = class {
109
198
  this.stopLipSyncAnalysis();
110
199
  this.audioAnalysisStream = stream;
111
200
  try {
112
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
- if (!audioContextConstructor) {
114
- throw new Error("AudioContext is not supported in this browser");
201
+ if (!this.audioContext) {
202
+ void this.prepareAudioContext().catch((error) => {
203
+ this.options.onError?.(error);
204
+ });
205
+ }
206
+ if (!this.audioContext) {
207
+ return;
115
208
  }
116
- this.audioContext = new audioContextConstructor();
117
- const source = this.audioContext.createMediaStreamSource(stream);
209
+ this.audioSource?.disconnect?.();
210
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
118
211
  this.analyser = this.audioContext.createAnalyser();
119
212
  this.analyser.fftSize = 256;
120
213
  this.analyser.smoothingTimeConstant = 0.8;
121
- source.connect(this.analyser);
214
+ this.audioSource.connect(this.analyser);
122
215
  this.startLipSyncAnalysis();
123
216
  } catch (error) {
124
217
  this.options.onError?.(error);
@@ -127,10 +220,21 @@ var LipSyncAnalyzer = class {
127
220
  stopOutput() {
128
221
  this.stopLipSyncAnalysis();
129
222
  }
223
+ pause() {
224
+ this.stopLipSyncAnalysis();
225
+ }
226
+ resume() {
227
+ if (!this.analyser || this.lipSyncInterval) {
228
+ return;
229
+ }
230
+ this.startLipSyncAnalysis();
231
+ }
130
232
  cleanup() {
131
233
  this.stopLipSyncAnalysis();
132
234
  this.stopAudioElementPolling();
133
235
  this.detachObservedAudioElement();
236
+ this.audioSource?.disconnect?.();
237
+ this.audioSource = null;
134
238
  if (this.audioContext) {
135
239
  void this.audioContext.close();
136
240
  this.audioContext = null;
@@ -183,18 +287,20 @@ var LipSyncAnalyzer = class {
183
287
  }
184
288
  };
185
289
 
290
+ // src/instructions.ts
291
+ var DEFAULT_REALTIME_VOICE = "marin";
292
+
186
293
  // src/openai-agents/session-config.ts
187
294
  var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
188
295
  var DEFAULT_MODEL = "gpt-realtime-mini";
189
- var DEFAULT_VOICE = "marin";
190
296
  function resolveInstructions(config) {
191
297
  return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
192
298
  }
193
299
  function resolveVoice(config) {
194
- return config?.voice ?? DEFAULT_VOICE;
300
+ return config?.voice ?? DEFAULT_REALTIME_VOICE;
195
301
  }
196
302
  function toOpenAIRealtimeAgentsSessionConfig(config) {
197
- return {
303
+ const sessionConfig = {
198
304
  model: config?.model ?? DEFAULT_MODEL,
199
305
  instructions: resolveInstructions(config),
200
306
  toolChoice: config?.toolChoice ?? "auto",
@@ -205,6 +311,13 @@ function toOpenAIRealtimeAgentsSessionConfig(config) {
205
311
  }
206
312
  }
207
313
  };
314
+ if (config?.temperature !== void 0) {
315
+ sessionConfig.temperature = config.temperature;
316
+ }
317
+ if (config?.maxTokens !== void 0) {
318
+ sessionConfig.maxResponseOutputTokens = config.maxTokens;
319
+ }
320
+ return sessionConfig;
208
321
  }
209
322
 
210
323
  // src/openai-agents/tool-schema.ts
@@ -256,9 +369,21 @@ var OpenAIRealtimeAgentsClient = class {
256
369
  session = null;
257
370
  transport = null;
258
371
  audioElement = null;
372
+ mediaStream = null;
373
+ peerConnection = null;
374
+ audioSender = null;
259
375
  connectionWasActive = false;
376
+ connectionLossNotified = false;
377
+ isExplicitDisconnect = false;
378
+ isRecovering = false;
379
+ isCleaningUp = false;
260
380
  assistant = { text: "", started: false };
261
381
  latestHistory = [];
382
+ currentSessionConfig;
383
+ teardownTransportLifecycle;
384
+ iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
385
+ this.emitConnectionLost("ice-disconnected");
386
+ }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
262
387
  eventCallbacks = /* @__PURE__ */ new Set();
263
388
  pendingToolCalls = /* @__PURE__ */ new Map();
264
389
  lipSyncAnalyzer = new LipSyncAnalyzer({
@@ -271,18 +396,23 @@ var OpenAIRealtimeAgentsClient = class {
271
396
  });
272
397
  async connect(config) {
273
398
  try {
399
+ if (this.isExplicitDisconnect && this.isRecovering) {
400
+ throw new Error("Realtime session disconnect requested");
401
+ }
274
402
  this.log("Starting OpenAI Agents Realtime WebRTC connection");
403
+ this.isExplicitDisconnect = false;
404
+ this.isRecovering = false;
405
+ this.connectionLossNotified = false;
406
+ this.currentSessionConfig = config;
275
407
  this.audioElement = document.createElement("audio");
276
408
  this.audioElement.autoplay = true;
277
409
  this.audioElement.setAttribute("playsinline", "true");
278
- const agent = new agentsRealtime.RealtimeAgent({
279
- name: "charivo-realtime-agent",
280
- instructions: resolveInstructions(config),
281
- tools: this.createProxyTools(config?.tools),
282
- voice: resolveVoice(config)
283
- });
410
+ this.mediaStream = await acquireMicrophoneStream();
411
+ this.bindTransportLifecycleEvents();
412
+ await this.lipSyncAnalyzer.prepareAudioContext();
413
+ const agent = this.createAgent(config);
284
414
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
285
- this.createTransportOptions(this.audioElement)
415
+ this.createTransportOptions(this.audioElement, this.mediaStream)
286
416
  );
287
417
  this.session = new agentsRealtime.RealtimeSession(agent, {
288
418
  transport: this.transport,
@@ -314,9 +444,37 @@ var OpenAIRealtimeAgentsClient = class {
314
444
  }
315
445
  async disconnect() {
316
446
  this.log("Disconnecting OpenAI Agents Realtime WebRTC");
447
+ this.isExplicitDisconnect = true;
448
+ this.isRecovering = false;
317
449
  this.cleanup();
318
450
  }
451
+ async updateSession(config) {
452
+ if (!this.session) {
453
+ throw new Error("Realtime session not active");
454
+ }
455
+ this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
456
+ await this.session.updateAgent(this.createAgent(config));
457
+ this.currentSessionConfig = config;
458
+ }
459
+ async recover(config) {
460
+ this.isRecovering = true;
461
+ this.connectionLossNotified = true;
462
+ this.currentSessionConfig = config;
463
+ try {
464
+ this.cleanupPendingToolCalls(
465
+ new Error("Realtime session interrupted during reconnect")
466
+ );
467
+ this.cleanup();
468
+ await this.connect(config ?? this.currentSessionConfig);
469
+ this.connectionLossNotified = false;
470
+ } finally {
471
+ this.isRecovering = false;
472
+ }
473
+ }
319
474
  async sendText(text) {
475
+ if (this.isRecovering) {
476
+ throw new Error("Realtime transport reconnecting");
477
+ }
320
478
  if (!this.session) {
321
479
  throw new Error("Realtime session not active");
322
480
  }
@@ -324,11 +482,17 @@ var OpenAIRealtimeAgentsClient = class {
324
482
  this.session.sendMessage(text);
325
483
  }
326
484
  async sendAudio(_audio) {
485
+ if (this.isRecovering) {
486
+ throw new Error("Realtime transport reconnecting");
487
+ }
327
488
  console.warn(
328
489
  "sendAudio is not needed with WebRTC - audio is automatically transmitted"
329
490
  );
330
491
  }
331
492
  async sendToolResult(callId, output) {
493
+ if (this.isRecovering) {
494
+ throw new Error("Realtime transport reconnecting");
495
+ }
332
496
  const pendingCall = this.pendingToolCalls.get(callId);
333
497
  if (!pendingCall) {
334
498
  throw new Error(`No pending realtime tool call for "${callId}"`);
@@ -337,6 +501,9 @@ var OpenAIRealtimeAgentsClient = class {
337
501
  pendingCall.resolve(output);
338
502
  }
339
503
  async interrupt() {
504
+ if (this.isRecovering) {
505
+ throw new Error("Realtime transport reconnecting");
506
+ }
340
507
  if (!this.session) {
341
508
  throw new Error("Realtime session not active");
342
509
  }
@@ -346,6 +513,14 @@ var OpenAIRealtimeAgentsClient = class {
346
513
  onEvent(callback) {
347
514
  this.eventCallbacks.add(callback);
348
515
  }
516
+ createAgent(config) {
517
+ return new agentsRealtime.RealtimeAgent({
518
+ name: "charivo-realtime-agent",
519
+ instructions: resolveInstructions(config),
520
+ tools: this.createProxyTools(config?.tools),
521
+ voice: resolveVoice(config)
522
+ });
523
+ }
349
524
  bindSessionEvents(session) {
350
525
  session.on("agent_start", () => {
351
526
  this.ensureAssistantStarted();
@@ -392,7 +567,7 @@ var OpenAIRealtimeAgentsClient = class {
392
567
  if (status === "disconnected" && this.connectionWasActive) {
393
568
  this.cleanupPendingToolCalls();
394
569
  this.connectionWasActive = false;
395
- this.emitEvent({ type: "session.ended" });
570
+ this.emitConnectionLost("connection-failed");
396
571
  }
397
572
  });
398
573
  }
@@ -494,10 +669,14 @@ var OpenAIRealtimeAgentsClient = class {
494
669
  });
495
670
  });
496
671
  }
497
- createTransportOptions(audioElement) {
672
+ createTransportOptions(audioElement, mediaStream) {
498
673
  return {
499
674
  audioElement,
675
+ mediaStream,
500
676
  changePeerConnection: async (peerConnection) => {
677
+ this.peerConnection = peerConnection;
678
+ this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
679
+ this.bindPeerConnectionEvents(peerConnection);
501
680
  peerConnection.addEventListener("track", (event) => {
502
681
  const stream = event.streams[0];
503
682
  if (stream) {
@@ -508,7 +687,113 @@ var OpenAIRealtimeAgentsClient = class {
508
687
  }
509
688
  };
510
689
  }
690
+ bindPeerConnectionEvents(peerConnection) {
691
+ peerConnection.addEventListener("iceconnectionstatechange", () => {
692
+ const iceState = peerConnection.iceConnectionState;
693
+ if (iceState === "failed") {
694
+ this.iceDisconnectDebouncer.cancel();
695
+ this.emitConnectionLost("ice-failed");
696
+ } else if (iceState === "disconnected") {
697
+ this.iceDisconnectDebouncer.schedule();
698
+ } else {
699
+ this.iceDisconnectDebouncer.cancel();
700
+ }
701
+ });
702
+ peerConnection.addEventListener("connectionstatechange", () => {
703
+ if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
704
+ this.iceDisconnectDebouncer.cancel();
705
+ this.emitConnectionLost("connection-failed");
706
+ }
707
+ });
708
+ }
709
+ bindTransportLifecycleEvents() {
710
+ if (this.teardownTransportLifecycle) {
711
+ return;
712
+ }
713
+ this.teardownTransportLifecycle = bindTransportLifecycle({
714
+ onHidden: this.handleHidden,
715
+ onOnline: this.handleOnline,
716
+ onPageHide: this.handlePageHide,
717
+ onPageShow: this.handlePageShow,
718
+ onVisible: this.handleVisible,
719
+ onDeviceChange: this.handleDeviceChange
720
+ });
721
+ }
722
+ unbindTransportLifecycleEvents() {
723
+ this.teardownTransportLifecycle?.();
724
+ this.teardownTransportLifecycle = void 0;
725
+ }
726
+ handleOnline = () => {
727
+ this.emitConnectionLost("online");
728
+ };
729
+ handleHidden = () => {
730
+ this.lipSyncAnalyzer.pause();
731
+ };
732
+ handleVisible = () => {
733
+ this.lipSyncAnalyzer.resume();
734
+ this.emitConnectionLost("visibility");
735
+ };
736
+ handlePageHide = () => {
737
+ this.lipSyncAnalyzer.pause();
738
+ };
739
+ handlePageShow = (event) => {
740
+ this.lipSyncAnalyzer.resume();
741
+ if (event.persisted) {
742
+ this.emitConnectionLost("pageshow");
743
+ }
744
+ };
745
+ handleDeviceChange = () => {
746
+ void this.refreshMicrophoneTrack();
747
+ };
748
+ async refreshMicrophoneTrack() {
749
+ if (!this.peerConnection || this.isExplicitDisconnect) {
750
+ return;
751
+ }
752
+ try {
753
+ const { audioSender, mediaStream } = await replaceMicrophoneTrack({
754
+ audioSender: this.audioSender,
755
+ mediaStream: this.mediaStream,
756
+ peerConnection: this.peerConnection
757
+ });
758
+ this.audioSender = audioSender;
759
+ this.mediaStream = mediaStream;
760
+ } catch (error) {
761
+ this.emitEvent({
762
+ type: "error",
763
+ error: error instanceof Error ? error : new Error(String(error))
764
+ });
765
+ }
766
+ }
767
+ emitConnectionLost(cause, error) {
768
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
769
+ return;
770
+ }
771
+ this.connectionLossNotified = true;
772
+ this.cleanupPendingToolCalls(
773
+ new Error("Realtime session interrupted during reconnect")
774
+ );
775
+ this.resetAssistantTracking();
776
+ this.lipSyncAnalyzer.stopOutput();
777
+ this.emitEvent({
778
+ type: "connection.lost",
779
+ cause,
780
+ error
781
+ });
782
+ }
783
+ shouldRecover(cause) {
784
+ if (!this.peerConnection) {
785
+ return false;
786
+ }
787
+ if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
788
+ return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
789
+ }
790
+ return true;
791
+ }
511
792
  cleanup(error) {
793
+ this.connectionWasActive = false;
794
+ this.isCleaningUp = true;
795
+ this.iceDisconnectDebouncer.cancel();
796
+ this.unbindTransportLifecycleEvents();
512
797
  this.cleanupPendingToolCalls(error);
513
798
  this.lipSyncAnalyzer.cleanup();
514
799
  if (this.transport) {
@@ -523,7 +808,15 @@ var OpenAIRealtimeAgentsClient = class {
523
808
  this.audioElement.srcObject = null;
524
809
  this.audioElement = null;
525
810
  }
811
+ if (this.mediaStream) {
812
+ this.mediaStream.getTracks?.().forEach((track) => track.stop());
813
+ this.mediaStream = null;
814
+ }
815
+ this.peerConnection = null;
816
+ this.audioSender = null;
526
817
  this.latestHistory = [];
818
+ this.connectionLossNotified = false;
819
+ this.isCleaningUp = false;
527
820
  this.resetAssistantTracking();
528
821
  }
529
822
  resetAssistantTracking() {