@charivo/realtime 0.7.2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -312,16 +312,25 @@ function resolveVoice(config) {
312
312
  return config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE;
313
313
  }
314
314
  function toOpenAIRealtimeAgentsSessionConfig(config) {
315
+ const audio = {
316
+ output: {
317
+ voice: resolveVoice(config)
318
+ }
319
+ };
320
+ const transcription = config?.inputAudioTranscription;
321
+ if (transcription !== void 0) {
322
+ if (transcription.enabled === false) {
323
+ audio.input = { transcription: null };
324
+ } else if (transcription.model !== void 0) {
325
+ audio.input = { transcription: { model: transcription.model } };
326
+ }
327
+ }
315
328
  const sessionConfig = {
316
329
  model: config?.model ?? DEFAULT_OPENAI_REALTIME_MODEL,
317
330
  instructions: resolveInstructions(config),
318
331
  toolChoice: config?.toolChoice ?? "auto",
319
332
  outputModalities: ["audio"],
320
- audio: {
321
- output: {
322
- voice: resolveVoice(config)
323
- }
324
- }
333
+ audio
325
334
  };
326
335
  if (config?.temperature !== void 0) {
327
336
  sessionConfig.temperature = config.temperature;
@@ -875,6 +884,15 @@ var OpenAIRealtimeAgentsClient = class {
875
884
  function createOpenAIRealtimeAgentsClient(options) {
876
885
  return new OpenAIRealtimeAgentsClient(options);
877
886
  }
887
+
888
+ // src/internal/timing.ts
889
+ function delay(ms) {
890
+ return new Promise((resolve) => {
891
+ setTimeout(resolve, ms);
892
+ });
893
+ }
894
+
895
+ // src/openai/client.ts
878
896
  var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
879
897
  "session.created",
880
898
  "session.updated",
@@ -894,10 +912,10 @@ var OpenAIRealtimeClient = class {
894
912
  audioElement = null;
895
913
  mediaStream = null;
896
914
  audioSender = null;
897
- audioContext = null;
898
- audioSource = null;
899
- analyser = null;
900
- lipSyncInterval = null;
915
+ lipSyncAnalyzer = new LipSyncAnalyzer({
916
+ onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
917
+ onError: (error) => console.error("Failed to setup audio analysis:", error)
918
+ });
901
919
  isResponseInProgress = false;
902
920
  cancelInFlight = false;
903
921
  hasStartedAssistantResponse = false;
@@ -929,14 +947,14 @@ var OpenAIRealtimeClient = class {
929
947
  this.audioElement = document.createElement("audio");
930
948
  this.audioElement.autoplay = true;
931
949
  this.audioElement.setAttribute("playsinline", "true");
932
- this.prepareAudioAnalysis();
950
+ void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
933
951
  this.bindConnectionEvents();
934
952
  this.bindTransportLifecycleEvents();
935
953
  this.pc.ontrack = (event) => {
936
954
  if (this.audioElement) {
937
955
  this.audioElement.srcObject = event.streams[0];
938
956
  }
939
- this.setupAudioAnalysis(event.streams[0]);
957
+ this.lipSyncAnalyzer.attachStream(event.streams[0]);
940
958
  };
941
959
  try {
942
960
  this.mediaStream = await acquireMicrophoneStream();
@@ -963,14 +981,10 @@ var OpenAIRealtimeClient = class {
963
981
  session: config ?? {},
964
982
  sdpOffer: offer.sdp
965
983
  });
966
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
967
- throw new Error(
968
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
969
- );
970
- }
984
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
971
985
  await this.pc.setRemoteDescription({
972
986
  type: "answer",
973
- sdp: bootstrap.answerSdp
987
+ sdp: answerSdp
974
988
  });
975
989
  this.emitEvent({ type: "session.started" });
976
990
  } catch (error) {
@@ -1159,10 +1173,7 @@ var OpenAIRealtimeClient = class {
1159
1173
  if (this.cancelInFlight || !event.delta) {
1160
1174
  return;
1161
1175
  }
1162
- if (!this.hasStartedAssistantResponse) {
1163
- this.hasStartedAssistantResponse = true;
1164
- this.emitEvent({ type: "assistant.response.started" });
1165
- }
1176
+ this.ensureAssistantResponseStarted();
1166
1177
  this.assistantText += event.delta;
1167
1178
  this.emitEvent({
1168
1179
  type: "assistant.text.delta",
@@ -1173,39 +1184,13 @@ var OpenAIRealtimeClient = class {
1173
1184
  if (this.cancelInFlight || !event.text) {
1174
1185
  return;
1175
1186
  }
1176
- if (!this.hasStartedAssistantResponse) {
1177
- this.hasStartedAssistantResponse = true;
1178
- this.emitEvent({ type: "assistant.response.started" });
1179
- }
1180
- if (event.text !== this.assistantText) {
1181
- const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
1182
- this.assistantText = event.text;
1183
- if (delta) {
1184
- this.emitEvent({
1185
- type: "assistant.text.delta",
1186
- text: delta
1187
- });
1188
- }
1189
- }
1187
+ this.emitFinalAssistantText(event.text);
1190
1188
  return;
1191
1189
  case "response.output_audio_transcript.done":
1192
1190
  if (this.cancelInFlight || !event.transcript) {
1193
1191
  return;
1194
1192
  }
1195
- if (!this.hasStartedAssistantResponse) {
1196
- this.hasStartedAssistantResponse = true;
1197
- this.emitEvent({ type: "assistant.response.started" });
1198
- }
1199
- if (event.transcript !== this.assistantText) {
1200
- const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
1201
- this.assistantText = event.transcript;
1202
- if (delta) {
1203
- this.emitEvent({
1204
- type: "assistant.text.delta",
1205
- text: delta
1206
- });
1207
- }
1208
- }
1193
+ this.emitFinalAssistantText(event.transcript);
1209
1194
  return;
1210
1195
  case "conversation.item.input_audio_transcription.completed":
1211
1196
  if (event.transcript) {
@@ -1336,17 +1321,17 @@ var OpenAIRealtimeClient = class {
1336
1321
  this.emitConnectionLost("online");
1337
1322
  };
1338
1323
  handleHidden = () => {
1339
- this.pauseAudioAnalysis();
1324
+ this.lipSyncAnalyzer.pause();
1340
1325
  };
1341
1326
  handleVisible = () => {
1342
- this.resumeAudioAnalysis();
1327
+ this.lipSyncAnalyzer.resume();
1343
1328
  this.emitConnectionLost("visibility");
1344
1329
  };
1345
1330
  handlePageHide = () => {
1346
- this.pauseAudioAnalysis();
1331
+ this.lipSyncAnalyzer.pause();
1347
1332
  };
1348
1333
  handlePageShow = (event) => {
1349
- this.resumeAudioAnalysis();
1334
+ this.lipSyncAnalyzer.resume();
1350
1335
  if (event.persisted) {
1351
1336
  this.emitConnectionLost("pageshow");
1352
1337
  }
@@ -1381,7 +1366,7 @@ var OpenAIRealtimeClient = class {
1381
1366
  this.isResponseInProgress = false;
1382
1367
  this.cancelInFlight = false;
1383
1368
  this.resetResponseTracking();
1384
- this.stopLipSyncAnalysis();
1369
+ this.lipSyncAnalyzer.stopOutput();
1385
1370
  this.emitEvent({
1386
1371
  type: "connection.lost",
1387
1372
  cause,
@@ -1397,77 +1382,11 @@ var OpenAIRealtimeClient = class {
1397
1382
  }
1398
1383
  return true;
1399
1384
  }
1400
- prepareAudioAnalysis() {
1401
- if (this.audioContext) {
1402
- return;
1403
- }
1404
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
1405
- if (!audioContextConstructor) {
1406
- return;
1407
- }
1408
- this.audioContext = new audioContextConstructor();
1409
- }
1410
- setupAudioAnalysis(stream) {
1411
- try {
1412
- this.prepareAudioAnalysis();
1413
- if (!this.audioContext) {
1414
- return;
1415
- }
1416
- this.audioSource?.disconnect();
1417
- this.audioSource = this.audioContext.createMediaStreamSource(stream);
1418
- this.analyser = this.audioContext.createAnalyser();
1419
- this.analyser.fftSize = 256;
1420
- this.analyser.smoothingTimeConstant = 0.8;
1421
- this.audioSource.connect(this.analyser);
1422
- this.startLipSyncAnalysis();
1423
- } catch (error) {
1424
- console.error("Failed to setup audio analysis:", error);
1425
- }
1426
- }
1427
- startLipSyncAnalysis() {
1428
- if (!this.analyser) {
1429
- return;
1430
- }
1431
- const bufferLength = this.analyser.frequencyBinCount;
1432
- const dataArray = new Uint8Array(bufferLength);
1433
- this.lipSyncInterval = window.setInterval(() => {
1434
- if (!this.analyser) {
1435
- return;
1436
- }
1437
- this.analyser.getByteFrequencyData(dataArray);
1438
- let sum = 0;
1439
- for (let index = 0; index < bufferLength; index += 1) {
1440
- const normalized = dataArray[index] / 255;
1441
- sum += normalized * normalized;
1442
- }
1443
- const rms = Math.sqrt(sum / bufferLength);
1444
- this.emitEvent({
1445
- type: "audio.lipsync",
1446
- rms: Math.min(rms * 3, 1)
1447
- });
1448
- }, 1e3 / 60);
1449
- }
1450
- stopLipSyncAnalysis() {
1451
- if (this.lipSyncInterval) {
1452
- clearInterval(this.lipSyncInterval);
1453
- this.lipSyncInterval = null;
1454
- }
1455
- this.emitEvent({ type: "audio.lipsync", rms: 0 });
1456
- }
1457
- pauseAudioAnalysis() {
1458
- this.stopLipSyncAnalysis();
1459
- }
1460
- resumeAudioAnalysis() {
1461
- if (!this.analyser || this.lipSyncInterval) {
1462
- return;
1463
- }
1464
- this.startLipSyncAnalysis();
1465
- }
1466
1385
  cleanup() {
1467
1386
  this.isCleaningUp = true;
1468
1387
  this.unbindTransportLifecycleEvents();
1469
1388
  this.iceDisconnectDebouncer.cancel();
1470
- this.stopLipSyncAnalysis();
1389
+ this.lipSyncAnalyzer.cleanup();
1471
1390
  this.rejectPendingSessionUpdate(
1472
1391
  new Error("Realtime session ended before session update completed")
1473
1392
  );
@@ -1487,13 +1406,7 @@ var OpenAIRealtimeClient = class {
1487
1406
  this.audioElement.srcObject = null;
1488
1407
  this.audioElement = null;
1489
1408
  }
1490
- if (this.audioContext) {
1491
- this.audioContext.close();
1492
- this.audioContext = null;
1493
- }
1494
- this.audioSource = null;
1495
1409
  this.audioSender = null;
1496
- this.analyser = null;
1497
1410
  this.isResponseInProgress = false;
1498
1411
  this.cancelInFlight = false;
1499
1412
  this.connectionLossNotified = false;
@@ -1541,14 +1454,10 @@ var OpenAIRealtimeClient = class {
1541
1454
  session: config ?? this.currentSessionConfig ?? {},
1542
1455
  sdpOffer: offer.sdp
1543
1456
  });
1544
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1545
- throw new Error(
1546
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1547
- );
1548
- }
1457
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
1549
1458
  await pc.setRemoteDescription({
1550
1459
  type: "answer",
1551
- sdp: bootstrap.answerSdp
1460
+ sdp: answerSdp
1552
1461
  });
1553
1462
  return true;
1554
1463
  }
@@ -1586,11 +1495,38 @@ var OpenAIRealtimeClient = class {
1586
1495
  getSessionUpdateTimeoutMs() {
1587
1496
  return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
1588
1497
  }
1498
+ resolveWebRTCAnswerSdp(bootstrap) {
1499
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
1500
+ throw new Error(
1501
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1502
+ );
1503
+ }
1504
+ return bootstrap.answerSdp;
1505
+ }
1589
1506
  resetResponseTracking() {
1590
1507
  this.assistantText = "";
1591
1508
  this.hasStartedAssistantResponse = false;
1592
1509
  this.hasStartedAudioOutput = false;
1593
1510
  }
1511
+ ensureAssistantResponseStarted() {
1512
+ if (!this.hasStartedAssistantResponse) {
1513
+ this.hasStartedAssistantResponse = true;
1514
+ this.emitEvent({ type: "assistant.response.started" });
1515
+ }
1516
+ }
1517
+ emitFinalAssistantText(full) {
1518
+ this.ensureAssistantResponseStarted();
1519
+ if (full !== this.assistantText) {
1520
+ const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
1521
+ this.assistantText = full;
1522
+ if (delta) {
1523
+ this.emitEvent({
1524
+ type: "assistant.text.delta",
1525
+ text: delta
1526
+ });
1527
+ }
1528
+ }
1529
+ }
1594
1530
  emitEvent(event) {
1595
1531
  for (const callback of this.eventCallbacks) {
1596
1532
  callback(event);
@@ -1612,12 +1548,21 @@ function createOpenAIRealtimeClient(options) {
1612
1548
  return new OpenAIRealtimeClient(options);
1613
1549
  }
1614
1550
  function toOpenAIRealtimeSessionUpdate(config) {
1551
+ const audio = {
1552
+ output: {
1553
+ voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
1554
+ }
1555
+ };
1556
+ const transcription = config?.inputAudioTranscription;
1557
+ if (transcription !== void 0) {
1558
+ if (transcription.enabled === false) {
1559
+ audio.input = { transcription: null };
1560
+ } else if (transcription.model !== void 0) {
1561
+ audio.input = { transcription: { model: transcription.model } };
1562
+ }
1563
+ }
1615
1564
  const session = {
1616
- audio: {
1617
- output: {
1618
- voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
1619
- }
1620
- },
1565
+ audio,
1621
1566
  tool_choice: config?.toolChoice ?? "auto"
1622
1567
  };
1623
1568
  if (config?.instructions) {
@@ -1629,16 +1574,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
1629
1574
  if (config?.maxTokens !== void 0) {
1630
1575
  session.max_response_output_tokens = config.maxTokens;
1631
1576
  }
1632
- if (config?.tools?.length) {
1633
- session.tools = config.tools;
1577
+ if (config && "tools" in config) {
1578
+ session.tools = config.tools ?? [];
1634
1579
  }
1635
1580
  return session;
1636
1581
  }
1637
- function delay(ms) {
1638
- return new Promise((resolve) => {
1639
- setTimeout(resolve, ms);
1640
- });
1641
- }
1642
1582
 
1643
1583
  // src/remote/index.ts
1644
1584
  var DEFAULT_REMOTE_REALTIME_ADAPTERS = {