@charivo/realtime 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -400,7 +400,7 @@ var OpenAIRealtimeAgentsClient = class {
400
400
  isCleaningUp = false;
401
401
  assistant = { text: "", started: false };
402
402
  assistantCompletionMetadata = {};
403
- latestHistory = [];
403
+ latestAssistantText = "";
404
404
  currentSessionConfig;
405
405
  teardownTransportLifecycle;
406
406
  iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
@@ -557,7 +557,7 @@ var OpenAIRealtimeAgentsClient = class {
557
557
  this.emitEvent({ type: "audio.output.ended" });
558
558
  });
559
559
  session.on("history_updated", (history) => {
560
- this.latestHistory = history;
560
+ this.latestAssistantText = this.extractLatestAssistantText(history);
561
561
  });
562
562
  session.on("error", ({ error }) => {
563
563
  this.emitEvent({
@@ -622,7 +622,7 @@ var OpenAIRealtimeAgentsClient = class {
622
622
  if (!this.assistant.text && !output.trim()) {
623
623
  return;
624
624
  }
625
- const finalText = this.getLatestAssistantText() || output || this.assistant.text;
625
+ const finalText = this.latestAssistantText || output || this.assistant.text;
626
626
  this.ensureAssistantStarted();
627
627
  if (finalText && finalText.startsWith(this.assistant.text) && finalText !== this.assistant.text) {
628
628
  const delta = finalText.slice(this.assistant.text.length);
@@ -640,9 +640,9 @@ var OpenAIRealtimeAgentsClient = class {
640
640
  });
641
641
  this.resetAssistantTracking();
642
642
  }
643
- getLatestAssistantText() {
644
- for (let index = this.latestHistory.length - 1; index >= 0; index -= 1) {
645
- const item = this.latestHistory[index];
643
+ extractLatestAssistantText(history) {
644
+ for (let index = history.length - 1; index >= 0; index -= 1) {
645
+ const item = history[index];
646
646
  if (item.type === "message" && item.role === "assistant" && Array.isArray(item.content)) {
647
647
  const text = item.content.map((content) => {
648
648
  if (content.type === "output_audio") {
@@ -844,7 +844,7 @@ var OpenAIRealtimeAgentsClient = class {
844
844
  }
845
845
  this.peerConnection = null;
846
846
  this.audioSender = null;
847
- this.latestHistory = [];
847
+ this.latestAssistantText = "";
848
848
  this.connectionLossNotified = false;
849
849
  this.isCleaningUp = false;
850
850
  this.resetAssistantTracking();
@@ -884,6 +884,15 @@ var OpenAIRealtimeAgentsClient = class {
884
884
  function createOpenAIRealtimeAgentsClient(options) {
885
885
  return new OpenAIRealtimeAgentsClient(options);
886
886
  }
887
+
888
+ // src/internal/timing.ts
889
+ function delay(ms) {
890
+ return new Promise((resolve) => {
891
+ setTimeout(resolve, ms);
892
+ });
893
+ }
894
+
895
+ // src/openai/client.ts
887
896
  var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
888
897
  "session.created",
889
898
  "session.updated",
@@ -903,10 +912,10 @@ var OpenAIRealtimeClient = class {
903
912
  audioElement = null;
904
913
  mediaStream = null;
905
914
  audioSender = null;
906
- audioContext = null;
907
- audioSource = null;
908
- analyser = null;
909
- lipSyncInterval = null;
915
+ lipSyncAnalyzer = new LipSyncAnalyzer({
916
+ onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
917
+ onError: (error) => console.error("Failed to setup audio analysis:", error)
918
+ });
910
919
  isResponseInProgress = false;
911
920
  cancelInFlight = false;
912
921
  hasStartedAssistantResponse = false;
@@ -938,14 +947,14 @@ var OpenAIRealtimeClient = class {
938
947
  this.audioElement = document.createElement("audio");
939
948
  this.audioElement.autoplay = true;
940
949
  this.audioElement.setAttribute("playsinline", "true");
941
- this.prepareAudioAnalysis();
950
+ void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
942
951
  this.bindConnectionEvents();
943
952
  this.bindTransportLifecycleEvents();
944
953
  this.pc.ontrack = (event) => {
945
954
  if (this.audioElement) {
946
955
  this.audioElement.srcObject = event.streams[0];
947
956
  }
948
- this.setupAudioAnalysis(event.streams[0]);
957
+ this.lipSyncAnalyzer.attachStream(event.streams[0]);
949
958
  };
950
959
  try {
951
960
  this.mediaStream = await acquireMicrophoneStream();
@@ -972,14 +981,10 @@ var OpenAIRealtimeClient = class {
972
981
  session: config ?? {},
973
982
  sdpOffer: offer.sdp
974
983
  });
975
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
976
- throw new Error(
977
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
978
- );
979
- }
984
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
980
985
  await this.pc.setRemoteDescription({
981
986
  type: "answer",
982
- sdp: bootstrap.answerSdp
987
+ sdp: answerSdp
983
988
  });
984
989
  this.emitEvent({ type: "session.started" });
985
990
  } catch (error) {
@@ -1168,10 +1173,7 @@ var OpenAIRealtimeClient = class {
1168
1173
  if (this.cancelInFlight || !event.delta) {
1169
1174
  return;
1170
1175
  }
1171
- if (!this.hasStartedAssistantResponse) {
1172
- this.hasStartedAssistantResponse = true;
1173
- this.emitEvent({ type: "assistant.response.started" });
1174
- }
1176
+ this.ensureAssistantResponseStarted();
1175
1177
  this.assistantText += event.delta;
1176
1178
  this.emitEvent({
1177
1179
  type: "assistant.text.delta",
@@ -1182,39 +1184,13 @@ var OpenAIRealtimeClient = class {
1182
1184
  if (this.cancelInFlight || !event.text) {
1183
1185
  return;
1184
1186
  }
1185
- if (!this.hasStartedAssistantResponse) {
1186
- this.hasStartedAssistantResponse = true;
1187
- this.emitEvent({ type: "assistant.response.started" });
1188
- }
1189
- if (event.text !== this.assistantText) {
1190
- const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
1191
- this.assistantText = event.text;
1192
- if (delta) {
1193
- this.emitEvent({
1194
- type: "assistant.text.delta",
1195
- text: delta
1196
- });
1197
- }
1198
- }
1187
+ this.emitFinalAssistantText(event.text);
1199
1188
  return;
1200
1189
  case "response.output_audio_transcript.done":
1201
1190
  if (this.cancelInFlight || !event.transcript) {
1202
1191
  return;
1203
1192
  }
1204
- if (!this.hasStartedAssistantResponse) {
1205
- this.hasStartedAssistantResponse = true;
1206
- this.emitEvent({ type: "assistant.response.started" });
1207
- }
1208
- if (event.transcript !== this.assistantText) {
1209
- const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
1210
- this.assistantText = event.transcript;
1211
- if (delta) {
1212
- this.emitEvent({
1213
- type: "assistant.text.delta",
1214
- text: delta
1215
- });
1216
- }
1217
- }
1193
+ this.emitFinalAssistantText(event.transcript);
1218
1194
  return;
1219
1195
  case "conversation.item.input_audio_transcription.completed":
1220
1196
  if (event.transcript) {
@@ -1345,17 +1321,17 @@ var OpenAIRealtimeClient = class {
1345
1321
  this.emitConnectionLost("online");
1346
1322
  };
1347
1323
  handleHidden = () => {
1348
- this.pauseAudioAnalysis();
1324
+ this.lipSyncAnalyzer.pause();
1349
1325
  };
1350
1326
  handleVisible = () => {
1351
- this.resumeAudioAnalysis();
1327
+ this.lipSyncAnalyzer.resume();
1352
1328
  this.emitConnectionLost("visibility");
1353
1329
  };
1354
1330
  handlePageHide = () => {
1355
- this.pauseAudioAnalysis();
1331
+ this.lipSyncAnalyzer.pause();
1356
1332
  };
1357
1333
  handlePageShow = (event) => {
1358
- this.resumeAudioAnalysis();
1334
+ this.lipSyncAnalyzer.resume();
1359
1335
  if (event.persisted) {
1360
1336
  this.emitConnectionLost("pageshow");
1361
1337
  }
@@ -1390,7 +1366,7 @@ var OpenAIRealtimeClient = class {
1390
1366
  this.isResponseInProgress = false;
1391
1367
  this.cancelInFlight = false;
1392
1368
  this.resetResponseTracking();
1393
- this.stopLipSyncAnalysis();
1369
+ this.lipSyncAnalyzer.stopOutput();
1394
1370
  this.emitEvent({
1395
1371
  type: "connection.lost",
1396
1372
  cause,
@@ -1406,77 +1382,11 @@ var OpenAIRealtimeClient = class {
1406
1382
  }
1407
1383
  return true;
1408
1384
  }
1409
- prepareAudioAnalysis() {
1410
- if (this.audioContext) {
1411
- return;
1412
- }
1413
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
1414
- if (!audioContextConstructor) {
1415
- return;
1416
- }
1417
- this.audioContext = new audioContextConstructor();
1418
- }
1419
- setupAudioAnalysis(stream) {
1420
- try {
1421
- this.prepareAudioAnalysis();
1422
- if (!this.audioContext) {
1423
- return;
1424
- }
1425
- this.audioSource?.disconnect();
1426
- this.audioSource = this.audioContext.createMediaStreamSource(stream);
1427
- this.analyser = this.audioContext.createAnalyser();
1428
- this.analyser.fftSize = 256;
1429
- this.analyser.smoothingTimeConstant = 0.8;
1430
- this.audioSource.connect(this.analyser);
1431
- this.startLipSyncAnalysis();
1432
- } catch (error) {
1433
- console.error("Failed to setup audio analysis:", error);
1434
- }
1435
- }
1436
- startLipSyncAnalysis() {
1437
- if (!this.analyser) {
1438
- return;
1439
- }
1440
- const bufferLength = this.analyser.frequencyBinCount;
1441
- const dataArray = new Uint8Array(bufferLength);
1442
- this.lipSyncInterval = window.setInterval(() => {
1443
- if (!this.analyser) {
1444
- return;
1445
- }
1446
- this.analyser.getByteFrequencyData(dataArray);
1447
- let sum = 0;
1448
- for (let index = 0; index < bufferLength; index += 1) {
1449
- const normalized = dataArray[index] / 255;
1450
- sum += normalized * normalized;
1451
- }
1452
- const rms = Math.sqrt(sum / bufferLength);
1453
- this.emitEvent({
1454
- type: "audio.lipsync",
1455
- rms: Math.min(rms * 3, 1)
1456
- });
1457
- }, 1e3 / 60);
1458
- }
1459
- stopLipSyncAnalysis() {
1460
- if (this.lipSyncInterval) {
1461
- clearInterval(this.lipSyncInterval);
1462
- this.lipSyncInterval = null;
1463
- }
1464
- this.emitEvent({ type: "audio.lipsync", rms: 0 });
1465
- }
1466
- pauseAudioAnalysis() {
1467
- this.stopLipSyncAnalysis();
1468
- }
1469
- resumeAudioAnalysis() {
1470
- if (!this.analyser || this.lipSyncInterval) {
1471
- return;
1472
- }
1473
- this.startLipSyncAnalysis();
1474
- }
1475
1385
  cleanup() {
1476
1386
  this.isCleaningUp = true;
1477
1387
  this.unbindTransportLifecycleEvents();
1478
1388
  this.iceDisconnectDebouncer.cancel();
1479
- this.stopLipSyncAnalysis();
1389
+ this.lipSyncAnalyzer.cleanup();
1480
1390
  this.rejectPendingSessionUpdate(
1481
1391
  new Error("Realtime session ended before session update completed")
1482
1392
  );
@@ -1496,13 +1406,7 @@ var OpenAIRealtimeClient = class {
1496
1406
  this.audioElement.srcObject = null;
1497
1407
  this.audioElement = null;
1498
1408
  }
1499
- if (this.audioContext) {
1500
- this.audioContext.close();
1501
- this.audioContext = null;
1502
- }
1503
- this.audioSource = null;
1504
1409
  this.audioSender = null;
1505
- this.analyser = null;
1506
1410
  this.isResponseInProgress = false;
1507
1411
  this.cancelInFlight = false;
1508
1412
  this.connectionLossNotified = false;
@@ -1550,14 +1454,10 @@ var OpenAIRealtimeClient = class {
1550
1454
  session: config ?? this.currentSessionConfig ?? {},
1551
1455
  sdpOffer: offer.sdp
1552
1456
  });
1553
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1554
- throw new Error(
1555
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1556
- );
1557
- }
1457
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
1558
1458
  await pc.setRemoteDescription({
1559
1459
  type: "answer",
1560
- sdp: bootstrap.answerSdp
1460
+ sdp: answerSdp
1561
1461
  });
1562
1462
  return true;
1563
1463
  }
@@ -1595,11 +1495,38 @@ var OpenAIRealtimeClient = class {
1595
1495
  getSessionUpdateTimeoutMs() {
1596
1496
  return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
1597
1497
  }
1498
+ resolveWebRTCAnswerSdp(bootstrap) {
1499
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1500
+ throw new Error(
1501
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1502
+ );
1503
+ }
1504
+ return bootstrap.answerSdp;
1505
+ }
1598
1506
  resetResponseTracking() {
1599
1507
  this.assistantText = "";
1600
1508
  this.hasStartedAssistantResponse = false;
1601
1509
  this.hasStartedAudioOutput = false;
1602
1510
  }
1511
+ ensureAssistantResponseStarted() {
1512
+ if (!this.hasStartedAssistantResponse) {
1513
+ this.hasStartedAssistantResponse = true;
1514
+ this.emitEvent({ type: "assistant.response.started" });
1515
+ }
1516
+ }
1517
+ emitFinalAssistantText(full) {
1518
+ this.ensureAssistantResponseStarted();
1519
+ if (full !== this.assistantText) {
1520
+ const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
1521
+ this.assistantText = full;
1522
+ if (delta) {
1523
+ this.emitEvent({
1524
+ type: "assistant.text.delta",
1525
+ text: delta
1526
+ });
1527
+ }
1528
+ }
1529
+ }
1603
1530
  emitEvent(event) {
1604
1531
  for (const callback of this.eventCallbacks) {
1605
1532
  callback(event);
@@ -1652,11 +1579,6 @@ function toOpenAIRealtimeSessionUpdate(config) {
1652
1579
  }
1653
1580
  return session;
1654
1581
  }
1655
- function delay(ms) {
1656
- return new Promise((resolve) => {
1657
- setTimeout(resolve, ms);
1658
- });
1659
- }
1660
1582
 
1661
1583
  // src/remote/index.ts
1662
1584
  var DEFAULT_REMOTE_REALTIME_ADAPTERS = {