@charivo/realtime 0.7.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/chunk-3PIMXNQU.mjs +222 -0
- package/dist/chunk-3PIMXNQU.mjs.map +1 -0
- package/dist/{chunk-FRTYFCPV.mjs → chunk-MI4QTRWT.mjs} +67 -144
- package/dist/chunk-MI4QTRWT.mjs.map +1 -0
- package/dist/{chunk-X6YRHDTW.mjs → chunk-PNBYFAER.mjs} +4 -3
- package/dist/chunk-PNBYFAER.mjs.map +1 -0
- package/dist/{chunk-GJ2NA3ZT.mjs → chunk-VSKMCGWD.mjs} +18 -142
- package/dist/chunk-VSKMCGWD.mjs.map +1 -0
- package/dist/chunk-XPKFJWHL.mjs +10 -0
- package/dist/chunk-XPKFJWHL.mjs.map +1 -0
- package/dist/index.d.mts +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +52 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +53 -10
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +4 -10
- package/dist/openai/index.d.ts +4 -10
- package/dist/openai/index.js +202 -140
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +4 -3
- package/dist/openai-agents/index.js +14 -5
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.js +85 -145
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +5 -4
- package/dist/remote/index.mjs.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-E656Y2PO.mjs +0 -89
- package/dist/chunk-E656Y2PO.mjs.map +0 -1
- package/dist/chunk-FRTYFCPV.mjs.map +0 -1
- package/dist/chunk-GJ2NA3ZT.mjs.map +0 -1
- package/dist/chunk-X6YRHDTW.mjs.map +0 -1
package/dist/remote/index.js
CHANGED
|
@@ -312,16 +312,25 @@ function resolveVoice(config) {
|
|
|
312
312
|
return config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE;
|
|
313
313
|
}
|
|
314
314
|
function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
315
|
+
const audio = {
|
|
316
|
+
output: {
|
|
317
|
+
voice: resolveVoice(config)
|
|
318
|
+
}
|
|
319
|
+
};
|
|
320
|
+
const transcription = config?.inputAudioTranscription;
|
|
321
|
+
if (transcription !== void 0) {
|
|
322
|
+
if (transcription.enabled === false) {
|
|
323
|
+
audio.input = { transcription: null };
|
|
324
|
+
} else if (transcription.model !== void 0) {
|
|
325
|
+
audio.input = { transcription: { model: transcription.model } };
|
|
326
|
+
}
|
|
327
|
+
}
|
|
315
328
|
const sessionConfig = {
|
|
316
329
|
model: config?.model ?? DEFAULT_OPENAI_REALTIME_MODEL,
|
|
317
330
|
instructions: resolveInstructions(config),
|
|
318
331
|
toolChoice: config?.toolChoice ?? "auto",
|
|
319
332
|
outputModalities: ["audio"],
|
|
320
|
-
audio
|
|
321
|
-
output: {
|
|
322
|
-
voice: resolveVoice(config)
|
|
323
|
-
}
|
|
324
|
-
}
|
|
333
|
+
audio
|
|
325
334
|
};
|
|
326
335
|
if (config?.temperature !== void 0) {
|
|
327
336
|
sessionConfig.temperature = config.temperature;
|
|
@@ -875,6 +884,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
875
884
|
function createOpenAIRealtimeAgentsClient(options) {
|
|
876
885
|
return new OpenAIRealtimeAgentsClient(options);
|
|
877
886
|
}
|
|
887
|
+
|
|
888
|
+
// src/internal/timing.ts
|
|
889
|
+
function delay(ms) {
|
|
890
|
+
return new Promise((resolve) => {
|
|
891
|
+
setTimeout(resolve, ms);
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// src/openai/client.ts
|
|
878
896
|
var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
|
|
879
897
|
"session.created",
|
|
880
898
|
"session.updated",
|
|
@@ -894,10 +912,10 @@ var OpenAIRealtimeClient = class {
|
|
|
894
912
|
audioElement = null;
|
|
895
913
|
mediaStream = null;
|
|
896
914
|
audioSender = null;
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
915
|
+
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
916
|
+
onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
|
|
917
|
+
onError: (error) => console.error("Failed to setup audio analysis:", error)
|
|
918
|
+
});
|
|
901
919
|
isResponseInProgress = false;
|
|
902
920
|
cancelInFlight = false;
|
|
903
921
|
hasStartedAssistantResponse = false;
|
|
@@ -929,14 +947,14 @@ var OpenAIRealtimeClient = class {
|
|
|
929
947
|
this.audioElement = document.createElement("audio");
|
|
930
948
|
this.audioElement.autoplay = true;
|
|
931
949
|
this.audioElement.setAttribute("playsinline", "true");
|
|
932
|
-
this.
|
|
950
|
+
void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
|
|
933
951
|
this.bindConnectionEvents();
|
|
934
952
|
this.bindTransportLifecycleEvents();
|
|
935
953
|
this.pc.ontrack = (event) => {
|
|
936
954
|
if (this.audioElement) {
|
|
937
955
|
this.audioElement.srcObject = event.streams[0];
|
|
938
956
|
}
|
|
939
|
-
this.
|
|
957
|
+
this.lipSyncAnalyzer.attachStream(event.streams[0]);
|
|
940
958
|
};
|
|
941
959
|
try {
|
|
942
960
|
this.mediaStream = await acquireMicrophoneStream();
|
|
@@ -963,14 +981,10 @@ var OpenAIRealtimeClient = class {
|
|
|
963
981
|
session: config ?? {},
|
|
964
982
|
sdpOffer: offer.sdp
|
|
965
983
|
});
|
|
966
|
-
|
|
967
|
-
throw new Error(
|
|
968
|
-
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
969
|
-
);
|
|
970
|
-
}
|
|
984
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
971
985
|
await this.pc.setRemoteDescription({
|
|
972
986
|
type: "answer",
|
|
973
|
-
sdp:
|
|
987
|
+
sdp: answerSdp
|
|
974
988
|
});
|
|
975
989
|
this.emitEvent({ type: "session.started" });
|
|
976
990
|
} catch (error) {
|
|
@@ -1159,10 +1173,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1159
1173
|
if (this.cancelInFlight || !event.delta) {
|
|
1160
1174
|
return;
|
|
1161
1175
|
}
|
|
1162
|
-
|
|
1163
|
-
this.hasStartedAssistantResponse = true;
|
|
1164
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1165
|
-
}
|
|
1176
|
+
this.ensureAssistantResponseStarted();
|
|
1166
1177
|
this.assistantText += event.delta;
|
|
1167
1178
|
this.emitEvent({
|
|
1168
1179
|
type: "assistant.text.delta",
|
|
@@ -1173,39 +1184,13 @@ var OpenAIRealtimeClient = class {
|
|
|
1173
1184
|
if (this.cancelInFlight || !event.text) {
|
|
1174
1185
|
return;
|
|
1175
1186
|
}
|
|
1176
|
-
|
|
1177
|
-
this.hasStartedAssistantResponse = true;
|
|
1178
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1179
|
-
}
|
|
1180
|
-
if (event.text !== this.assistantText) {
|
|
1181
|
-
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
1182
|
-
this.assistantText = event.text;
|
|
1183
|
-
if (delta) {
|
|
1184
|
-
this.emitEvent({
|
|
1185
|
-
type: "assistant.text.delta",
|
|
1186
|
-
text: delta
|
|
1187
|
-
});
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1187
|
+
this.emitFinalAssistantText(event.text);
|
|
1190
1188
|
return;
|
|
1191
1189
|
case "response.output_audio_transcript.done":
|
|
1192
1190
|
if (this.cancelInFlight || !event.transcript) {
|
|
1193
1191
|
return;
|
|
1194
1192
|
}
|
|
1195
|
-
|
|
1196
|
-
this.hasStartedAssistantResponse = true;
|
|
1197
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
1198
|
-
}
|
|
1199
|
-
if (event.transcript !== this.assistantText) {
|
|
1200
|
-
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
1201
|
-
this.assistantText = event.transcript;
|
|
1202
|
-
if (delta) {
|
|
1203
|
-
this.emitEvent({
|
|
1204
|
-
type: "assistant.text.delta",
|
|
1205
|
-
text: delta
|
|
1206
|
-
});
|
|
1207
|
-
}
|
|
1208
|
-
}
|
|
1193
|
+
this.emitFinalAssistantText(event.transcript);
|
|
1209
1194
|
return;
|
|
1210
1195
|
case "conversation.item.input_audio_transcription.completed":
|
|
1211
1196
|
if (event.transcript) {
|
|
@@ -1336,17 +1321,17 @@ var OpenAIRealtimeClient = class {
|
|
|
1336
1321
|
this.emitConnectionLost("online");
|
|
1337
1322
|
};
|
|
1338
1323
|
handleHidden = () => {
|
|
1339
|
-
this.
|
|
1324
|
+
this.lipSyncAnalyzer.pause();
|
|
1340
1325
|
};
|
|
1341
1326
|
handleVisible = () => {
|
|
1342
|
-
this.
|
|
1327
|
+
this.lipSyncAnalyzer.resume();
|
|
1343
1328
|
this.emitConnectionLost("visibility");
|
|
1344
1329
|
};
|
|
1345
1330
|
handlePageHide = () => {
|
|
1346
|
-
this.
|
|
1331
|
+
this.lipSyncAnalyzer.pause();
|
|
1347
1332
|
};
|
|
1348
1333
|
handlePageShow = (event) => {
|
|
1349
|
-
this.
|
|
1334
|
+
this.lipSyncAnalyzer.resume();
|
|
1350
1335
|
if (event.persisted) {
|
|
1351
1336
|
this.emitConnectionLost("pageshow");
|
|
1352
1337
|
}
|
|
@@ -1381,7 +1366,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1381
1366
|
this.isResponseInProgress = false;
|
|
1382
1367
|
this.cancelInFlight = false;
|
|
1383
1368
|
this.resetResponseTracking();
|
|
1384
|
-
this.
|
|
1369
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
1385
1370
|
this.emitEvent({
|
|
1386
1371
|
type: "connection.lost",
|
|
1387
1372
|
cause,
|
|
@@ -1397,77 +1382,11 @@ var OpenAIRealtimeClient = class {
|
|
|
1397
1382
|
}
|
|
1398
1383
|
return true;
|
|
1399
1384
|
}
|
|
1400
|
-
prepareAudioAnalysis() {
|
|
1401
|
-
if (this.audioContext) {
|
|
1402
|
-
return;
|
|
1403
|
-
}
|
|
1404
|
-
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
1405
|
-
if (!audioContextConstructor) {
|
|
1406
|
-
return;
|
|
1407
|
-
}
|
|
1408
|
-
this.audioContext = new audioContextConstructor();
|
|
1409
|
-
}
|
|
1410
|
-
setupAudioAnalysis(stream) {
|
|
1411
|
-
try {
|
|
1412
|
-
this.prepareAudioAnalysis();
|
|
1413
|
-
if (!this.audioContext) {
|
|
1414
|
-
return;
|
|
1415
|
-
}
|
|
1416
|
-
this.audioSource?.disconnect();
|
|
1417
|
-
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
1418
|
-
this.analyser = this.audioContext.createAnalyser();
|
|
1419
|
-
this.analyser.fftSize = 256;
|
|
1420
|
-
this.analyser.smoothingTimeConstant = 0.8;
|
|
1421
|
-
this.audioSource.connect(this.analyser);
|
|
1422
|
-
this.startLipSyncAnalysis();
|
|
1423
|
-
} catch (error) {
|
|
1424
|
-
console.error("Failed to setup audio analysis:", error);
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
startLipSyncAnalysis() {
|
|
1428
|
-
if (!this.analyser) {
|
|
1429
|
-
return;
|
|
1430
|
-
}
|
|
1431
|
-
const bufferLength = this.analyser.frequencyBinCount;
|
|
1432
|
-
const dataArray = new Uint8Array(bufferLength);
|
|
1433
|
-
this.lipSyncInterval = window.setInterval(() => {
|
|
1434
|
-
if (!this.analyser) {
|
|
1435
|
-
return;
|
|
1436
|
-
}
|
|
1437
|
-
this.analyser.getByteFrequencyData(dataArray);
|
|
1438
|
-
let sum = 0;
|
|
1439
|
-
for (let index = 0; index < bufferLength; index += 1) {
|
|
1440
|
-
const normalized = dataArray[index] / 255;
|
|
1441
|
-
sum += normalized * normalized;
|
|
1442
|
-
}
|
|
1443
|
-
const rms = Math.sqrt(sum / bufferLength);
|
|
1444
|
-
this.emitEvent({
|
|
1445
|
-
type: "audio.lipsync",
|
|
1446
|
-
rms: Math.min(rms * 3, 1)
|
|
1447
|
-
});
|
|
1448
|
-
}, 1e3 / 60);
|
|
1449
|
-
}
|
|
1450
|
-
stopLipSyncAnalysis() {
|
|
1451
|
-
if (this.lipSyncInterval) {
|
|
1452
|
-
clearInterval(this.lipSyncInterval);
|
|
1453
|
-
this.lipSyncInterval = null;
|
|
1454
|
-
}
|
|
1455
|
-
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
1456
|
-
}
|
|
1457
|
-
pauseAudioAnalysis() {
|
|
1458
|
-
this.stopLipSyncAnalysis();
|
|
1459
|
-
}
|
|
1460
|
-
resumeAudioAnalysis() {
|
|
1461
|
-
if (!this.analyser || this.lipSyncInterval) {
|
|
1462
|
-
return;
|
|
1463
|
-
}
|
|
1464
|
-
this.startLipSyncAnalysis();
|
|
1465
|
-
}
|
|
1466
1385
|
cleanup() {
|
|
1467
1386
|
this.isCleaningUp = true;
|
|
1468
1387
|
this.unbindTransportLifecycleEvents();
|
|
1469
1388
|
this.iceDisconnectDebouncer.cancel();
|
|
1470
|
-
this.
|
|
1389
|
+
this.lipSyncAnalyzer.cleanup();
|
|
1471
1390
|
this.rejectPendingSessionUpdate(
|
|
1472
1391
|
new Error("Realtime session ended before session update completed")
|
|
1473
1392
|
);
|
|
@@ -1487,13 +1406,7 @@ var OpenAIRealtimeClient = class {
|
|
|
1487
1406
|
this.audioElement.srcObject = null;
|
|
1488
1407
|
this.audioElement = null;
|
|
1489
1408
|
}
|
|
1490
|
-
if (this.audioContext) {
|
|
1491
|
-
this.audioContext.close();
|
|
1492
|
-
this.audioContext = null;
|
|
1493
|
-
}
|
|
1494
|
-
this.audioSource = null;
|
|
1495
1409
|
this.audioSender = null;
|
|
1496
|
-
this.analyser = null;
|
|
1497
1410
|
this.isResponseInProgress = false;
|
|
1498
1411
|
this.cancelInFlight = false;
|
|
1499
1412
|
this.connectionLossNotified = false;
|
|
@@ -1541,14 +1454,10 @@ var OpenAIRealtimeClient = class {
|
|
|
1541
1454
|
session: config ?? this.currentSessionConfig ?? {},
|
|
1542
1455
|
sdpOffer: offer.sdp
|
|
1543
1456
|
});
|
|
1544
|
-
|
|
1545
|
-
throw new Error(
|
|
1546
|
-
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1547
|
-
);
|
|
1548
|
-
}
|
|
1457
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
1549
1458
|
await pc.setRemoteDescription({
|
|
1550
1459
|
type: "answer",
|
|
1551
|
-
sdp:
|
|
1460
|
+
sdp: answerSdp
|
|
1552
1461
|
});
|
|
1553
1462
|
return true;
|
|
1554
1463
|
}
|
|
@@ -1586,11 +1495,38 @@ var OpenAIRealtimeClient = class {
|
|
|
1586
1495
|
getSessionUpdateTimeoutMs() {
|
|
1587
1496
|
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
1588
1497
|
}
|
|
1498
|
+
resolveWebRTCAnswerSdp(bootstrap) {
|
|
1499
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
1500
|
+
throw new Error(
|
|
1501
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1502
|
+
);
|
|
1503
|
+
}
|
|
1504
|
+
return bootstrap.answerSdp;
|
|
1505
|
+
}
|
|
1589
1506
|
resetResponseTracking() {
|
|
1590
1507
|
this.assistantText = "";
|
|
1591
1508
|
this.hasStartedAssistantResponse = false;
|
|
1592
1509
|
this.hasStartedAudioOutput = false;
|
|
1593
1510
|
}
|
|
1511
|
+
ensureAssistantResponseStarted() {
|
|
1512
|
+
if (!this.hasStartedAssistantResponse) {
|
|
1513
|
+
this.hasStartedAssistantResponse = true;
|
|
1514
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
emitFinalAssistantText(full) {
|
|
1518
|
+
this.ensureAssistantResponseStarted();
|
|
1519
|
+
if (full !== this.assistantText) {
|
|
1520
|
+
const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
|
|
1521
|
+
this.assistantText = full;
|
|
1522
|
+
if (delta) {
|
|
1523
|
+
this.emitEvent({
|
|
1524
|
+
type: "assistant.text.delta",
|
|
1525
|
+
text: delta
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1594
1530
|
emitEvent(event) {
|
|
1595
1531
|
for (const callback of this.eventCallbacks) {
|
|
1596
1532
|
callback(event);
|
|
@@ -1612,12 +1548,21 @@ function createOpenAIRealtimeClient(options) {
|
|
|
1612
1548
|
return new OpenAIRealtimeClient(options);
|
|
1613
1549
|
}
|
|
1614
1550
|
function toOpenAIRealtimeSessionUpdate(config) {
|
|
1551
|
+
const audio = {
|
|
1552
|
+
output: {
|
|
1553
|
+
voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
|
|
1554
|
+
}
|
|
1555
|
+
};
|
|
1556
|
+
const transcription = config?.inputAudioTranscription;
|
|
1557
|
+
if (transcription !== void 0) {
|
|
1558
|
+
if (transcription.enabled === false) {
|
|
1559
|
+
audio.input = { transcription: null };
|
|
1560
|
+
} else if (transcription.model !== void 0) {
|
|
1561
|
+
audio.input = { transcription: { model: transcription.model } };
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1615
1564
|
const session = {
|
|
1616
|
-
audio
|
|
1617
|
-
output: {
|
|
1618
|
-
voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
|
|
1619
|
-
}
|
|
1620
|
-
},
|
|
1565
|
+
audio,
|
|
1621
1566
|
tool_choice: config?.toolChoice ?? "auto"
|
|
1622
1567
|
};
|
|
1623
1568
|
if (config?.instructions) {
|
|
@@ -1629,16 +1574,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
1629
1574
|
if (config?.maxTokens !== void 0) {
|
|
1630
1575
|
session.max_response_output_tokens = config.maxTokens;
|
|
1631
1576
|
}
|
|
1632
|
-
if (config
|
|
1633
|
-
session.tools = config.tools;
|
|
1577
|
+
if (config && "tools" in config) {
|
|
1578
|
+
session.tools = config.tools ?? [];
|
|
1634
1579
|
}
|
|
1635
1580
|
return session;
|
|
1636
1581
|
}
|
|
1637
|
-
function delay(ms) {
|
|
1638
|
-
return new Promise((resolve) => {
|
|
1639
|
-
setTimeout(resolve, ms);
|
|
1640
|
-
});
|
|
1641
|
-
}
|
|
1642
1582
|
|
|
1643
1583
|
// src/remote/index.ts
|
|
1644
1584
|
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|