@charivo/realtime 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -132,7 +132,7 @@ Speak only the words you want heard aloud. Use tools for actions, but never ment
132
132
  `.trim();
133
133
 
134
134
  // src/openai/defaults.ts
135
- var DEFAULT_OPENAI_REALTIME_MODEL = "gpt-realtime-mini";
135
+ var DEFAULT_OPENAI_REALTIME_MODEL = "gpt-realtime-2.1-mini";
136
136
  var DEFAULT_OPENAI_REALTIME_VOICE = "marin";
137
137
  var DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
138
138
 
@@ -271,139 +271,6 @@ async function getOpenAIRealtimeAgentsBootstrap(options, request) {
271
271
  return bootstrap;
272
272
  }
273
273
 
274
- // src/openai-agents/lip-sync-analyzer.ts
275
- var LipSyncAnalyzer = class {
276
- constructor(options) {
277
- this.options = options;
278
- }
279
- audioAnalysisStream = null;
280
- audioContext = null;
281
- audioSource = null;
282
- analyser = null;
283
- lipSyncInterval = null;
284
- pendingAudioElementPoll = null;
285
- observedAudioElement = null;
286
- observedAudioElementListener = null;
287
- async prepareAudioContext() {
288
- if (this.audioContext) {
289
- return;
290
- }
291
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
292
- if (!audioContextConstructor) {
293
- throw new Error("AudioContext is not supported in this browser");
294
- }
295
- this.audioContext = new audioContextConstructor();
296
- }
297
- observeAudioElement(audioElement) {
298
- this.detachObservedAudioElement();
299
- const tryAttachStream = () => {
300
- const stream = audioElement.srcObject;
301
- if (stream instanceof MediaStream) {
302
- this.attachStream(stream);
303
- }
304
- };
305
- this.observedAudioElement = audioElement;
306
- this.observedAudioElementListener = tryAttachStream;
307
- audioElement.addEventListener("loadedmetadata", tryAttachStream);
308
- this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
309
- }
310
- attachStream(stream) {
311
- if (this.audioAnalysisStream === stream) {
312
- return;
313
- }
314
- this.stopAudioElementPolling();
315
- this.stopLipSyncAnalysis();
316
- this.audioAnalysisStream = stream;
317
- try {
318
- if (!this.audioContext) {
319
- void this.prepareAudioContext().catch((error) => {
320
- this.options.onError?.(error);
321
- });
322
- }
323
- if (!this.audioContext) {
324
- return;
325
- }
326
- this.audioSource?.disconnect?.();
327
- this.audioSource = this.audioContext.createMediaStreamSource(stream);
328
- this.analyser = this.audioContext.createAnalyser();
329
- this.analyser.fftSize = 256;
330
- this.analyser.smoothingTimeConstant = 0.8;
331
- this.audioSource.connect(this.analyser);
332
- this.startLipSyncAnalysis();
333
- } catch (error) {
334
- this.options.onError?.(error);
335
- }
336
- }
337
- stopOutput() {
338
- this.stopLipSyncAnalysis();
339
- }
340
- pause() {
341
- this.stopLipSyncAnalysis();
342
- }
343
- resume() {
344
- if (!this.analyser || this.lipSyncInterval) {
345
- return;
346
- }
347
- this.startLipSyncAnalysis();
348
- }
349
- cleanup() {
350
- this.stopLipSyncAnalysis();
351
- this.stopAudioElementPolling();
352
- this.detachObservedAudioElement();
353
- this.audioSource?.disconnect?.();
354
- this.audioSource = null;
355
- if (this.audioContext) {
356
- void this.audioContext.close();
357
- this.audioContext = null;
358
- }
359
- this.audioAnalysisStream = null;
360
- this.analyser = null;
361
- }
362
- startLipSyncAnalysis() {
363
- if (!this.analyser) {
364
- return;
365
- }
366
- const bufferLength = this.analyser.frequencyBinCount;
367
- const dataArray = new Uint8Array(bufferLength);
368
- this.lipSyncInterval = window.setInterval(() => {
369
- if (!this.analyser) {
370
- return;
371
- }
372
- this.analyser.getByteFrequencyData(dataArray);
373
- let sum = 0;
374
- for (let index = 0; index < bufferLength; index += 1) {
375
- const normalized = dataArray[index] / 255;
376
- sum += normalized * normalized;
377
- }
378
- const rms = Math.sqrt(sum / bufferLength);
379
- this.options.onRms(Math.min(rms * 3, 1));
380
- }, 1e3 / 60);
381
- }
382
- stopLipSyncAnalysis() {
383
- if (this.lipSyncInterval) {
384
- clearInterval(this.lipSyncInterval);
385
- this.lipSyncInterval = null;
386
- }
387
- this.options.onRms(0);
388
- }
389
- stopAudioElementPolling() {
390
- if (this.pendingAudioElementPoll) {
391
- clearInterval(this.pendingAudioElementPoll);
392
- this.pendingAudioElementPoll = null;
393
- }
394
- }
395
- detachObservedAudioElement() {
396
- if (this.observedAudioElement && this.observedAudioElementListener) {
397
- this.observedAudioElement.removeEventListener(
398
- "loadedmetadata",
399
- this.observedAudioElementListener
400
- );
401
- }
402
- this.observedAudioElement = null;
403
- this.observedAudioElementListener = null;
404
- }
405
- };
406
-
407
274
  // src/openai-agents/session-config.ts
408
275
  function resolveInstructions(config) {
409
276
  return config?.instructions ?? DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS;
@@ -508,7 +375,7 @@ var OpenAIRealtimeAgentsClient = class {
508
375
  }, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
509
376
  eventCallbacks = /* @__PURE__ */ new Set();
510
377
  pendingToolCalls = /* @__PURE__ */ new Map();
511
- lipSyncAnalyzer = new LipSyncAnalyzer({
378
+ lipSyncAnalyzer = core.createLipSyncAnalyzer({
512
379
  onRms: (rms) => {
513
380
  this.emitEvent({ type: "audio.lipsync", rms });
514
381
  },
@@ -516,6 +383,9 @@ var OpenAIRealtimeAgentsClient = class {
516
383
  console.error("Failed to setup audio analysis:", error);
517
384
  }
518
385
  });
386
+ observedAudioElement = null;
387
+ observedAudioElementListener = null;
388
+ pendingAudioElementPoll = null;
519
389
  async connect(config) {
520
390
  try {
521
391
  if (this.isExplicitDisconnect && this.isRecovering) {
@@ -531,7 +401,7 @@ var OpenAIRealtimeAgentsClient = class {
531
401
  this.audioElement.setAttribute("playsinline", "true");
532
402
  this.mediaStream = await acquireMicrophoneStream();
533
403
  this.bindTransportLifecycleEvents();
534
- await this.lipSyncAnalyzer.prepareAudioContext();
404
+ await this.lipSyncAnalyzer.prepare();
535
405
  const agent = this.createAgent(config);
536
406
  this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
537
407
  this.createTransportOptions(this.audioElement, this.mediaStream)
@@ -542,7 +412,7 @@ var OpenAIRealtimeAgentsClient = class {
542
412
  });
543
413
  this.bindSessionEvents(this.session);
544
414
  this.bindTransportEvents(this.transport);
545
- this.lipSyncAnalyzer.observeAudioElement(this.audioElement);
415
+ this.observeAudioElement(this.audioElement);
546
416
  const bootstrap = await this.getSessionBootstrap({
547
417
  adapter: core.OPENAI_REALTIME_AGENTS_ADAPTER,
548
418
  transport: "webrtc",
@@ -635,6 +505,9 @@ var OpenAIRealtimeAgentsClient = class {
635
505
  onEvent(callback) {
636
506
  this.eventCallbacks.add(callback);
637
507
  }
508
+ async prepareAudio() {
509
+ await this.lipSyncAnalyzer.prepare();
510
+ }
638
511
  createAgent(config) {
639
512
  return new agentsRealtime.RealtimeAgent({
640
513
  name: "charivo-realtime-agent",
@@ -651,6 +524,7 @@ var OpenAIRealtimeAgentsClient = class {
651
524
  this.finalizeAssistantResponse(output);
652
525
  });
653
526
  session.on("audio_start", () => {
527
+ this.lipSyncAnalyzer.resume();
654
528
  this.emitEvent({ type: "audio.output.started" });
655
529
  });
656
530
  session.on("audio_stopped", () => {
@@ -690,7 +564,7 @@ var OpenAIRealtimeAgentsClient = class {
690
564
  });
691
565
  transport.on("audio_interrupted", () => {
692
566
  this.emitEvent({ type: "audio.output.ended" });
693
- this.lipSyncAnalyzer.stopOutput();
567
+ this.lipSyncAnalyzer.pause();
694
568
  });
695
569
  transport.on("connection_change", (status) => {
696
570
  if (status === "disconnected" && this.connectionWasActive) {
@@ -810,7 +684,7 @@ var OpenAIRealtimeAgentsClient = class {
810
684
  peerConnection.addEventListener("track", (event) => {
811
685
  const stream = event.streams[0];
812
686
  if (stream) {
813
- this.lipSyncAnalyzer.attachStream(stream);
687
+ this.lipSyncAnalyzer.attachMediaStream(stream);
814
688
  }
815
689
  });
816
690
  return peerConnection;
@@ -894,6 +768,34 @@ var OpenAIRealtimeAgentsClient = class {
894
768
  });
895
769
  }
896
770
  }
771
+ observeAudioElement(audioElement) {
772
+ this.stopObservingAudioElement();
773
+ const tryAttachStream = () => {
774
+ const stream = audioElement.srcObject;
775
+ if (stream instanceof MediaStream) {
776
+ this.lipSyncAnalyzer.attachMediaStream(stream);
777
+ this.stopObservingAudioElement();
778
+ }
779
+ };
780
+ this.observedAudioElement = audioElement;
781
+ this.observedAudioElementListener = tryAttachStream;
782
+ audioElement.addEventListener("loadedmetadata", tryAttachStream);
783
+ this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
784
+ }
785
+ stopObservingAudioElement() {
786
+ if (this.pendingAudioElementPoll !== null) {
787
+ clearInterval(this.pendingAudioElementPoll);
788
+ this.pendingAudioElementPoll = null;
789
+ }
790
+ if (this.observedAudioElement && this.observedAudioElementListener) {
791
+ this.observedAudioElement.removeEventListener(
792
+ "loadedmetadata",
793
+ this.observedAudioElementListener
794
+ );
795
+ }
796
+ this.observedAudioElement = null;
797
+ this.observedAudioElementListener = null;
798
+ }
897
799
  emitConnectionLost(cause, error) {
898
800
  if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
899
801
  return;
@@ -903,7 +805,7 @@ var OpenAIRealtimeAgentsClient = class {
903
805
  new Error("Realtime session interrupted during reconnect")
904
806
  );
905
807
  this.resetAssistantTracking();
906
- this.lipSyncAnalyzer.stopOutput();
808
+ this.lipSyncAnalyzer.stop();
907
809
  this.emitEvent({
908
810
  type: "connection.lost",
909
811
  cause,
@@ -925,7 +827,10 @@ var OpenAIRealtimeAgentsClient = class {
925
827
  this.iceDisconnectDebouncer.cancel();
926
828
  this.unbindTransportLifecycleEvents();
927
829
  this.cleanupPendingToolCalls(error);
928
- this.lipSyncAnalyzer.cleanup();
830
+ this.stopObservingAudioElement();
831
+ this.lipSyncAnalyzer.cleanup().catch(
832
+ (cleanupError) => console.error("Failed to clean up lip-sync analyzer:", cleanupError)
833
+ );
929
834
  if (this.transport) {
930
835
  this.transport.close();
931
836
  this.transport = null;
@@ -1012,7 +917,7 @@ var OpenAIRealtimeClient = class {
1012
917
  audioElement = null;
1013
918
  mediaStream = null;
1014
919
  audioSender = null;
1015
- lipSyncAnalyzer = new LipSyncAnalyzer({
920
+ lipSyncAnalyzer = core.createLipSyncAnalyzer({
1016
921
  onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
1017
922
  onError: (error) => console.error("Failed to setup audio analysis:", error)
1018
923
  });
@@ -1047,14 +952,14 @@ var OpenAIRealtimeClient = class {
1047
952
  this.audioElement = document.createElement("audio");
1048
953
  this.audioElement.autoplay = true;
1049
954
  this.audioElement.setAttribute("playsinline", "true");
1050
- void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
955
+ void this.lipSyncAnalyzer.prepare().catch(() => void 0);
1051
956
  this.bindConnectionEvents();
1052
957
  this.bindTransportLifecycleEvents();
1053
958
  this.pc.ontrack = (event) => {
1054
959
  if (this.audioElement) {
1055
960
  this.audioElement.srcObject = event.streams[0];
1056
961
  }
1057
- this.lipSyncAnalyzer.attachStream(event.streams[0]);
962
+ this.lipSyncAnalyzer.attachMediaStream(event.streams[0]);
1058
963
  };
1059
964
  try {
1060
965
  this.mediaStream = await acquireMicrophoneStream();
@@ -1218,6 +1123,9 @@ var OpenAIRealtimeClient = class {
1218
1123
  onEvent(callback) {
1219
1124
  this.eventCallbacks.add(callback);
1220
1125
  }
1126
+ async prepareAudio() {
1127
+ await this.lipSyncAnalyzer.prepare();
1128
+ }
1221
1129
  handleServerEvent(event) {
1222
1130
  if (this.options.debug && this.shouldLogDebugEvent(event.type)) {
1223
1131
  this.log("\u{1F4E1} [OpenAI Realtime Event]", event.type, event);
@@ -1466,7 +1374,7 @@ var OpenAIRealtimeClient = class {
1466
1374
  this.isResponseInProgress = false;
1467
1375
  this.cancelInFlight = false;
1468
1376
  this.resetResponseTracking();
1469
- this.lipSyncAnalyzer.stopOutput();
1377
+ this.lipSyncAnalyzer.stop();
1470
1378
  this.emitEvent({
1471
1379
  type: "connection.lost",
1472
1380
  cause,
@@ -1486,7 +1394,9 @@ var OpenAIRealtimeClient = class {
1486
1394
  this.isCleaningUp = true;
1487
1395
  this.unbindTransportLifecycleEvents();
1488
1396
  this.iceDisconnectDebouncer.cancel();
1489
- this.lipSyncAnalyzer.cleanup();
1397
+ this.lipSyncAnalyzer.cleanup().catch(
1398
+ (error) => console.error("Failed to clean up lip-sync analyzer:", error)
1399
+ );
1490
1400
  this.rejectPendingSessionUpdate(
1491
1401
  new Error("Realtime session ended before session update completed")
1492
1402
  );
@@ -1702,23 +1612,21 @@ var RemoteRealtimeClient = class {
1702
1612
  );
1703
1613
  }
1704
1614
  transportClient = null;
1615
+ preparedTransportClient = null;
1616
+ preparedAdapterId = null;
1705
1617
  adapters;
1706
1618
  eventCallbacks = /* @__PURE__ */ new Set();
1707
1619
  async connect(config) {
1708
1620
  try {
1709
1621
  const adapterId = this.resolveAdapterId(config);
1710
- const factory = this.adapters.get(adapterId);
1711
- if (!factory) {
1712
- throw new core.CharivoStateError(
1713
- `No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
1714
- );
1715
- }
1716
- const transportClient = factory({
1717
- debug: this.config.debug,
1718
- requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
1719
- });
1720
- for (const callback of this.eventCallbacks) {
1721
- transportClient.onEvent(callback);
1622
+ let transportClient;
1623
+ if (this.preparedAdapterId === adapterId && this.preparedTransportClient) {
1624
+ transportClient = this.preparedTransportClient;
1625
+ this.preparedTransportClient = null;
1626
+ this.preparedAdapterId = null;
1627
+ } else {
1628
+ void this.discardPreparedTransportClient();
1629
+ transportClient = this.createTransportClient(adapterId);
1722
1630
  }
1723
1631
  this.transportClient = transportClient;
1724
1632
  await transportClient.connect(config);
@@ -1727,6 +1635,64 @@ var RemoteRealtimeClient = class {
1727
1635
  throw core.toCharivoError("transport", error);
1728
1636
  }
1729
1637
  }
1638
+ /**
1639
+ * Creates and prepares the browser-side transport adapter for `config` up
1640
+ * front so its audio context/lip-sync analyzer is warmed inside the same
1641
+ * user-gesture call stack. `connect()` reuses this same adapter instance
1642
+ * when it is called with a matching adapter selection. Repeated calls that
1643
+ * resolve to the same adapter reuse the already-prepared instance instead
1644
+ * of creating another one.
1645
+ */
1646
+ async prepareAudio(config) {
1647
+ try {
1648
+ const adapterId = this.resolveAdapterId(config);
1649
+ if (this.preparedAdapterId !== adapterId || !this.preparedTransportClient) {
1650
+ void this.discardPreparedTransportClient();
1651
+ this.preparedTransportClient = this.createTransportClient(adapterId);
1652
+ this.preparedAdapterId = adapterId;
1653
+ }
1654
+ await this.preparedTransportClient.prepareAudio?.();
1655
+ } catch (error) {
1656
+ void this.discardPreparedTransportClient();
1657
+ throw core.toCharivoError("transport", error);
1658
+ }
1659
+ }
1660
+ /**
1661
+ * Releases a prepared-but-never-connected adapter's audio resources
1662
+ * (e.g. its `AudioContext`) when it is superseded, mismatched, or fails.
1663
+ * `disconnect()` on these adapters is safe to call without a prior
1664
+ * `connect()` — it tears down the lip-sync analyzer unconditionally.
1665
+ * Resolves once cleanup settles; a disconnect failure is reported instead
1666
+ * of being silently swallowed, since it was never part of an active
1667
+ * session and the caller has nothing else to react to.
1668
+ */
1669
+ discardPreparedTransportClient() {
1670
+ const prepared = this.preparedTransportClient;
1671
+ this.preparedTransportClient = null;
1672
+ this.preparedAdapterId = null;
1673
+ if (!prepared || prepared === this.transportClient) {
1674
+ return Promise.resolve();
1675
+ }
1676
+ return prepared.disconnect().catch((error) => {
1677
+ console.error("Failed to release a prepared realtime adapter:", error);
1678
+ });
1679
+ }
1680
+ createTransportClient(adapterId) {
1681
+ const factory = this.adapters.get(adapterId);
1682
+ if (!factory) {
1683
+ throw new core.CharivoStateError(
1684
+ `No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
1685
+ );
1686
+ }
1687
+ const transportClient = factory({
1688
+ debug: this.config.debug,
1689
+ requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
1690
+ });
1691
+ for (const callback of this.eventCallbacks) {
1692
+ transportClient.onEvent(callback);
1693
+ }
1694
+ return transportClient;
1695
+ }
1730
1696
  async updateSession(config) {
1731
1697
  await this.getActiveTransportClient().updateSession(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1732
1698
  }
@@ -1734,6 +1700,7 @@ var RemoteRealtimeClient = class {
1734
1700
  await this.getActiveTransportClient().recover(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
1735
1701
  }
1736
1702
  async disconnect() {
1703
+ await this.discardPreparedTransportClient();
1737
1704
  if (!this.transportClient) {
1738
1705
  return;
1739
1706
  }
@@ -1756,6 +1723,9 @@ var RemoteRealtimeClient = class {
1756
1723
  onEvent(callback) {
1757
1724
  this.eventCallbacks.add(callback);
1758
1725
  this.transportClient?.onEvent(callback);
1726
+ if (this.preparedTransportClient && this.preparedTransportClient !== this.transportClient) {
1727
+ this.preparedTransportClient.onEvent(callback);
1728
+ }
1759
1729
  }
1760
1730
  getActiveTransportClient() {
1761
1731
  if (!this.transportClient) {