@charivo/realtime 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-3PIMXNQU.mjs +222 -0
- package/dist/chunk-3PIMXNQU.mjs.map +1 -0
- package/dist/{chunk-VYMT3Z2W.mjs → chunk-FZ2LAUCO.mjs} +10 -143
- package/dist/chunk-FZ2LAUCO.mjs.map +1 -0
- package/dist/{chunk-QD4QSGPO.mjs → chunk-MI4QTRWT.mjs} +50 -136
- package/dist/chunk-MI4QTRWT.mjs.map +1 -0
- package/dist/chunk-XPKFJWHL.mjs +10 -0
- package/dist/chunk-XPKFJWHL.mjs.map +1 -0
- package/dist/index.mjs +1 -7
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +4 -10
- package/dist/openai/index.d.ts +4 -10
- package/dist/openai/index.js +186 -133
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.d.mts +2 -2
- package/dist/openai-agents/index.d.ts +2 -2
- package/dist/openai-agents/index.js +7 -7
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +2 -2
- package/dist/remote/index.js +62 -140
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -3
- package/dist/remote/index.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-QD4QSGPO.mjs.map +0 -1
- package/dist/chunk-VYMT3Z2W.mjs.map +0 -1
- package/dist/chunk-YQWOBEE5.mjs +0 -89
- package/dist/chunk-YQWOBEE5.mjs.map +0 -1
package/dist/remote/index.js
CHANGED
|
@@ -400,7 +400,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
400
400
|
isCleaningUp = false;
|
|
401
401
|
assistant = { text: "", started: false };
|
|
402
402
|
assistantCompletionMetadata = {};
|
|
403
|
-
|
|
403
|
+
latestAssistantText = "";
|
|
404
404
|
currentSessionConfig;
|
|
405
405
|
teardownTransportLifecycle;
|
|
406
406
|
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
@@ -557,7 +557,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
557
557
|
this.emitEvent({ type: "audio.output.ended" });
|
|
558
558
|
});
|
|
559
559
|
session.on("history_updated", (history) => {
|
|
560
|
-
this.
|
|
560
|
+
this.latestAssistantText = this.extractLatestAssistantText(history);
|
|
561
561
|
});
|
|
562
562
|
session.on("error", ({ error }) => {
|
|
563
563
|
this.emitEvent({
|
|
@@ -622,7 +622,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
622
622
|
if (!this.assistant.text && !output.trim()) {
|
|
623
623
|
return;
|
|
624
624
|
}
|
|
625
|
-
const finalText = this.
|
|
625
|
+
const finalText = this.latestAssistantText || output || this.assistant.text;
|
|
626
626
|
this.ensureAssistantStarted();
|
|
627
627
|
if (finalText && finalText.startsWith(this.assistant.text) && finalText !== this.assistant.text) {
|
|
628
628
|
const delta = finalText.slice(this.assistant.text.length);
|
|
@@ -640,9 +640,9 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
640
640
|
});
|
|
641
641
|
this.resetAssistantTracking();
|
|
642
642
|
}
|
|
643
|
-
|
|
644
|
-
for (let index =
|
|
645
|
-
const item =
|
|
643
|
+
extractLatestAssistantText(history) {
|
|
644
|
+
for (let index = history.length - 1; index >= 0; index -= 1) {
|
|
645
|
+
const item = history[index];
|
|
646
646
|
if (item.type === "message" && item.role === "assistant" && Array.isArray(item.content)) {
|
|
647
647
|
const text = item.content.map((content) => {
|
|
648
648
|
if (content.type === "output_audio") {
|
|
@@ -844,7 +844,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
844
844
|
}
|
|
845
845
|
this.peerConnection = null;
|
|
846
846
|
this.audioSender = null;
|
|
847
|
-
this.
|
|
847
|
+
this.latestAssistantText = "";
|
|
848
848
|
this.connectionLossNotified = false;
|
|
849
849
|
this.isCleaningUp = false;
|
|
850
850
|
this.resetAssistantTracking();
|
|
@@ -884,6 +884,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
884
884
|
function createOpenAIRealtimeAgentsClient(options) {
|
|
885
885
|
return new OpenAIRealtimeAgentsClient(options);
|
|
886
886
|
}
|
|
887
|
+
|
|
888
|
+
// src/internal/timing.ts
|
|
889
|
+
function delay(ms) {
|
|
890
|
+
return new Promise((resolve) => {
|
|
891
|
+
setTimeout(resolve, ms);
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// src/openai/client.ts
|
|
887
896
|
var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
|
|
888
897
|
"session.created",
|
|
889
898
|
"session.updated",
|
|
@@ -903,10 +912,10 @@ var OpenAIRealtimeClient = class {
|
|
|
903
912
|
audioElement = null;
|
|
904
913
|
mediaStream = null;
|
|
905
914
|
audioSender = null;
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
915
|
+
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
916
|
+
onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
|
|
917
|
+
onError: (error) => console.error("Failed to setup audio analysis:", error)
|
|
918
|
+
});
|
|
910
919
|
isResponseInProgress = false;
|
|
911
920
|
cancelInFlight = false;
|
|
912
921
|
hasStartedAssistantResponse = false;
|
|
@@ -938,14 +947,14 @@ var OpenAIRealtimeClient = class {
|
|
|
938
947
|
this.audioElement = document.createElement("audio");
|
|
939
948
|
this.audioElement.autoplay = true;
|
|
940
949
|
this.audioElement.setAttribute("playsinline", "true");
|
|
941
|
-
this.
|
|
950
|
+
void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
|
|
942
951
|
this.bindConnectionEvents();
|
|
943
952
|
this.bindTransportLifecycleEvents();
|
|
944
953
|
this.pc.ontrack = (event) => {
|
|
945
954
|
if (this.audioElement) {
|
|
946
955
|
this.audioElement.srcObject = event.streams[0];
|
|
947
956
|
}
|
|
948
|
-
this.
|
|
957
|
+
this.lipSyncAnalyzer.attachStream(event.streams[0]);
|
|
949
958
|
};
|
|
950
959
|
try {
|
|
951
960
|
this.mediaStream = await acquireMicrophoneStream();
|
|
@@ -972,14 +981,10 @@ var OpenAIRealtimeClient = class {
|
|
|
972
981
|
session: config ?? {},
|
|
973
982
|
sdpOffer: offer.sdp
|
|
974
983
|
});
|
|
975
|
-
|
|
976
|
-
throw new Error(
|
|
977
|
-
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
978
|
-
);
|
|
979
|
-
}
|
|
984
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
980
985
|
await this.pc.setRemoteDescription({
|
|
981
986
|
type: "answer",
|
|
982
|
-
sdp:
|
|
987
|
+
sdp: answerSdp
|
|
983
988
|
});
|
|
984
989
|
this.emitEvent({ type: "session.started" });
|
|
985
990
|
} catch (error) {
|
|
@@ -1168,10 +1173,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1168
1173
|
if (this.cancelInFlight || !event.delta) {
|
|
1169
1174
|
return;
|
|
1170
1175
|
}
|
|
1171
|
-
|
|
1172
|
-
this.hasStartedAssistantResponse = true;
|
|
1173
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1174
|
-
}
|
|
1176
|
+
this.ensureAssistantResponseStarted();
|
|
1175
1177
|
this.assistantText += event.delta;
|
|
1176
1178
|
this.emitEvent({
|
|
1177
1179
|
type: "assistant.text.delta",
|
|
@@ -1182,39 +1184,13 @@ var OpenAIRealtimeClient = class {
|
|
|
1182
1184
|
if (this.cancelInFlight || !event.text) {
|
|
1183
1185
|
return;
|
|
1184
1186
|
}
|
|
1185
|
-
|
|
1186
|
-
this.hasStartedAssistantResponse = true;
|
|
1187
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1188
|
-
}
|
|
1189
|
-
if (event.text !== this.assistantText) {
|
|
1190
|
-
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
1191
|
-
this.assistantText = event.text;
|
|
1192
|
-
if (delta) {
|
|
1193
|
-
this.emitEvent({
|
|
1194
|
-
type: "assistant.text.delta",
|
|
1195
|
-
text: delta
|
|
1196
|
-
});
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1187
|
+
this.emitFinalAssistantText(event.text);
|
|
1199
1188
|
return;
|
|
1200
1189
|
case "response.output_audio_transcript.done":
|
|
1201
1190
|
if (this.cancelInFlight || !event.transcript) {
|
|
1202
1191
|
return;
|
|
1203
1192
|
}
|
|
1204
|
-
|
|
1205
|
-
this.hasStartedAssistantResponse = true;
|
|
1206
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1207
|
-
}
|
|
1208
|
-
if (event.transcript !== this.assistantText) {
|
|
1209
|
-
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
1210
|
-
this.assistantText = event.transcript;
|
|
1211
|
-
if (delta) {
|
|
1212
|
-
this.emitEvent({
|
|
1213
|
-
type: "assistant.text.delta",
|
|
1214
|
-
text: delta
|
|
1215
|
-
});
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1193
|
+
this.emitFinalAssistantText(event.transcript);
|
|
1218
1194
|
return;
|
|
1219
1195
|
case "conversation.item.input_audio_transcription.completed":
|
|
1220
1196
|
if (event.transcript) {
|
|
@@ -1345,17 +1321,17 @@ var OpenAIRealtimeClient = class {
|
|
|
1345
1321
|
this.emitConnectionLost("online");
|
|
1346
1322
|
};
|
|
1347
1323
|
handleHidden = () => {
|
|
1348
|
-
this.
|
|
1324
|
+
this.lipSyncAnalyzer.pause();
|
|
1349
1325
|
};
|
|
1350
1326
|
handleVisible = () => {
|
|
1351
|
-
this.
|
|
1327
|
+
this.lipSyncAnalyzer.resume();
|
|
1352
1328
|
this.emitConnectionLost("visibility");
|
|
1353
1329
|
};
|
|
1354
1330
|
handlePageHide = () => {
|
|
1355
|
-
this.
|
|
1331
|
+
this.lipSyncAnalyzer.pause();
|
|
1356
1332
|
};
|
|
1357
1333
|
handlePageShow = (event) => {
|
|
1358
|
-
this.
|
|
1334
|
+
this.lipSyncAnalyzer.resume();
|
|
1359
1335
|
if (event.persisted) {
|
|
1360
1336
|
this.emitConnectionLost("pageshow");
|
|
1361
1337
|
}
|
|
@@ -1390,7 +1366,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1390
1366
|
this.isResponseInProgress = false;
|
|
1391
1367
|
this.cancelInFlight = false;
|
|
1392
1368
|
this.resetResponseTracking();
|
|
1393
|
-
this.
|
|
1369
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
1394
1370
|
this.emitEvent({
|
|
1395
1371
|
type: "connection.lost",
|
|
1396
1372
|
cause,
|
|
@@ -1406,77 +1382,11 @@ var OpenAIRealtimeClient = class {
|
|
|
1406
1382
|
}
|
|
1407
1383
|
return true;
|
|
1408
1384
|
}
|
|
1409
|
-
prepareAudioAnalysis() {
|
|
1410
|
-
if (this.audioContext) {
|
|
1411
|
-
return;
|
|
1412
|
-
}
|
|
1413
|
-
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
1414
|
-
if (!audioContextConstructor) {
|
|
1415
|
-
return;
|
|
1416
|
-
}
|
|
1417
|
-
this.audioContext = new audioContextConstructor();
|
|
1418
|
-
}
|
|
1419
|
-
setupAudioAnalysis(stream) {
|
|
1420
|
-
try {
|
|
1421
|
-
this.prepareAudioAnalysis();
|
|
1422
|
-
if (!this.audioContext) {
|
|
1423
|
-
return;
|
|
1424
|
-
}
|
|
1425
|
-
this.audioSource?.disconnect();
|
|
1426
|
-
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
1427
|
-
this.analyser = this.audioContext.createAnalyser();
|
|
1428
|
-
this.analyser.fftSize = 256;
|
|
1429
|
-
this.analyser.smoothingTimeConstant = 0.8;
|
|
1430
|
-
this.audioSource.connect(this.analyser);
|
|
1431
|
-
this.startLipSyncAnalysis();
|
|
1432
|
-
} catch (error) {
|
|
1433
|
-
console.error("Failed to setup audio analysis:", error);
|
|
1434
|
-
}
|
|
1435
|
-
}
|
|
1436
|
-
startLipSyncAnalysis() {
|
|
1437
|
-
if (!this.analyser) {
|
|
1438
|
-
return;
|
|
1439
|
-
}
|
|
1440
|
-
const bufferLength = this.analyser.frequencyBinCount;
|
|
1441
|
-
const dataArray = new Uint8Array(bufferLength);
|
|
1442
|
-
this.lipSyncInterval = window.setInterval(() => {
|
|
1443
|
-
if (!this.analyser) {
|
|
1444
|
-
return;
|
|
1445
|
-
}
|
|
1446
|
-
this.analyser.getByteFrequencyData(dataArray);
|
|
1447
|
-
let sum = 0;
|
|
1448
|
-
for (let index = 0; index < bufferLength; index += 1) {
|
|
1449
|
-
const normalized = dataArray[index] / 255;
|
|
1450
|
-
sum += normalized * normalized;
|
|
1451
|
-
}
|
|
1452
|
-
const rms = Math.sqrt(sum / bufferLength);
|
|
1453
|
-
this.emitEvent({
|
|
1454
|
-
type: "audio.lipsync",
|
|
1455
|
-
rms: Math.min(rms * 3, 1)
|
|
1456
|
-
});
|
|
1457
|
-
}, 1e3 / 60);
|
|
1458
|
-
}
|
|
1459
|
-
stopLipSyncAnalysis() {
|
|
1460
|
-
if (this.lipSyncInterval) {
|
|
1461
|
-
clearInterval(this.lipSyncInterval);
|
|
1462
|
-
this.lipSyncInterval = null;
|
|
1463
|
-
}
|
|
1464
|
-
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
1465
|
-
}
|
|
1466
|
-
pauseAudioAnalysis() {
|
|
1467
|
-
this.stopLipSyncAnalysis();
|
|
1468
|
-
}
|
|
1469
|
-
resumeAudioAnalysis() {
|
|
1470
|
-
if (!this.analyser || this.lipSyncInterval) {
|
|
1471
|
-
return;
|
|
1472
|
-
}
|
|
1473
|
-
this.startLipSyncAnalysis();
|
|
1474
|
-
}
|
|
1475
1385
|
cleanup() {
|
|
1476
1386
|
this.isCleaningUp = true;
|
|
1477
1387
|
this.unbindTransportLifecycleEvents();
|
|
1478
1388
|
this.iceDisconnectDebouncer.cancel();
|
|
1479
|
-
this.
|
|
1389
|
+
this.lipSyncAnalyzer.cleanup();
|
|
1480
1390
|
this.rejectPendingSessionUpdate(
|
|
1481
1391
|
new Error("Realtime session ended before session update completed")
|
|
1482
1392
|
);
|
|
@@ -1496,13 +1406,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1496
1406
|
this.audioElement.srcObject = null;
|
|
1497
1407
|
this.audioElement = null;
|
|
1498
1408
|
}
|
|
1499
|
-
if (this.audioContext) {
|
|
1500
|
-
this.audioContext.close();
|
|
1501
|
-
this.audioContext = null;
|
|
1502
|
-
}
|
|
1503
|
-
this.audioSource = null;
|
|
1504
1409
|
this.audioSender = null;
|
|
1505
|
-
this.analyser = null;
|
|
1506
1410
|
this.isResponseInProgress = false;
|
|
1507
1411
|
this.cancelInFlight = false;
|
|
1508
1412
|
this.connectionLossNotified = false;
|
|
@@ -1550,14 +1454,10 @@ var OpenAIRealtimeClient = class {
|
|
|
1550
1454
|
session: config ?? this.currentSessionConfig ?? {},
|
|
1551
1455
|
sdpOffer: offer.sdp
|
|
1552
1456
|
});
|
|
1553
|
-
|
|
1554
|
-
throw new Error(
|
|
1555
|
-
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1556
|
-
);
|
|
1557
|
-
}
|
|
1457
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
1558
1458
|
await pc.setRemoteDescription({
|
|
1559
1459
|
type: "answer",
|
|
1560
|
-
sdp:
|
|
1460
|
+
sdp: answerSdp
|
|
1561
1461
|
});
|
|
1562
1462
|
return true;
|
|
1563
1463
|
}
|
|
@@ -1595,11 +1495,38 @@ var OpenAIRealtimeClient = class {
|
|
|
1595
1495
|
getSessionUpdateTimeoutMs() {
|
|
1596
1496
|
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
1597
1497
|
}
|
|
1498
|
+
resolveWebRTCAnswerSdp(bootstrap) {
|
|
1499
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
1500
|
+
throw new Error(
|
|
1501
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1502
|
+
);
|
|
1503
|
+
}
|
|
1504
|
+
return bootstrap.answerSdp;
|
|
1505
|
+
}
|
|
1598
1506
|
resetResponseTracking() {
|
|
1599
1507
|
this.assistantText = "";
|
|
1600
1508
|
this.hasStartedAssistantResponse = false;
|
|
1601
1509
|
this.hasStartedAudioOutput = false;
|
|
1602
1510
|
}
|
|
1511
|
+
ensureAssistantResponseStarted() {
|
|
1512
|
+
if (!this.hasStartedAssistantResponse) {
|
|
1513
|
+
this.hasStartedAssistantResponse = true;
|
|
1514
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
emitFinalAssistantText(full) {
|
|
1518
|
+
this.ensureAssistantResponseStarted();
|
|
1519
|
+
if (full !== this.assistantText) {
|
|
1520
|
+
const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
|
|
1521
|
+
this.assistantText = full;
|
|
1522
|
+
if (delta) {
|
|
1523
|
+
this.emitEvent({
|
|
1524
|
+
type: "assistant.text.delta",
|
|
1525
|
+
text: delta
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1603
1530
|
emitEvent(event) {
|
|
1604
1531
|
for (const callback of this.eventCallbacks) {
|
|
1605
1532
|
callback(event);
|
|
@@ -1652,11 +1579,6 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
1652
1579
|
}
|
|
1653
1580
|
return session;
|
|
1654
1581
|
}
|
|
1655
|
-
function delay(ms) {
|
|
1656
|
-
return new Promise((resolve) => {
|
|
1657
|
-
setTimeout(resolve, ms);
|
|
1658
|
-
});
|
|
1659
|
-
}
|
|
1660
1582
|
|
|
1661
1583
|
// src/remote/index.ts
|
|
1662
1584
|
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|