@charivo/realtime 0.7.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/chunk-3PIMXNQU.mjs +222 -0
- package/dist/chunk-3PIMXNQU.mjs.map +1 -0
- package/dist/{chunk-FRTYFCPV.mjs → chunk-MI4QTRWT.mjs} +67 -144
- package/dist/chunk-MI4QTRWT.mjs.map +1 -0
- package/dist/{chunk-X6YRHDTW.mjs → chunk-PNBYFAER.mjs} +4 -3
- package/dist/chunk-PNBYFAER.mjs.map +1 -0
- package/dist/{chunk-GJ2NA3ZT.mjs → chunk-VSKMCGWD.mjs} +18 -142
- package/dist/chunk-VSKMCGWD.mjs.map +1 -0
- package/dist/chunk-XPKFJWHL.mjs +10 -0
- package/dist/chunk-XPKFJWHL.mjs.map +1 -0
- package/dist/index.d.mts +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +52 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +53 -10
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +4 -10
- package/dist/openai/index.d.ts +4 -10
- package/dist/openai/index.js +202 -140
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +4 -3
- package/dist/openai-agents/index.js +14 -5
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.js +85 -145
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +5 -4
- package/dist/remote/index.mjs.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-E656Y2PO.mjs +0 -89
- package/dist/chunk-E656Y2PO.mjs.map +0 -1
- package/dist/chunk-FRTYFCPV.mjs.map +0 -1
- package/dist/chunk-GJ2NA3ZT.mjs.map +0 -1
- package/dist/chunk-X6YRHDTW.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -50,10 +50,24 @@ await manager.startSession({
|
|
|
50
50
|
provider: "openai",
|
|
51
51
|
model: "gpt-realtime-mini",
|
|
52
52
|
});
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`gpt-realtime-mini` is the default realtime model; the full `gpt-realtime` is available but meaningfully more expensive — consult [OpenAI's pricing page](https://openai.com/api/pricing/) before switching.
|
|
53
56
|
|
|
57
|
+
```ts
|
|
54
58
|
await manager.updateSession({
|
|
55
59
|
voice: "alloy",
|
|
56
60
|
});
|
|
61
|
+
|
|
62
|
+
// Switch the user-transcription model (cost vs. quality), or disable it.
|
|
63
|
+
await manager.updateSession({
|
|
64
|
+
inputAudioTranscription: { model: "gpt-4o-mini-transcribe" },
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// Disable user transcription entirely (no whisper-1 charge):
|
|
68
|
+
await manager.updateSession({
|
|
69
|
+
inputAudioTranscription: { enabled: false },
|
|
70
|
+
});
|
|
57
71
|
```
|
|
58
72
|
|
|
59
73
|
If the live transport drops temporarily, `RealtimeManager` now keeps the
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS } from './chunk-PNBYFAER.mjs';
|
|
2
|
+
import { subscribeBrowserLifecycle } from '@charivo/core';
|
|
3
|
+
|
|
4
|
+
// src/internal/microphone.ts
|
|
5
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
6
|
+
echoCancellation: true,
|
|
7
|
+
noiseSuppression: true,
|
|
8
|
+
autoGainControl: true
|
|
9
|
+
};
|
|
10
|
+
async function acquireMicrophoneStream() {
|
|
11
|
+
return navigator.mediaDevices.getUserMedia({
|
|
12
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
16
|
+
function bindTransportLifecycle(callbacks) {
|
|
17
|
+
const teardownBrowserLifecycle = subscribeBrowserLifecycle({
|
|
18
|
+
onHidden: callbacks.onHidden,
|
|
19
|
+
onOnline: callbacks.onOnline,
|
|
20
|
+
onPageHide: callbacks.onPageHide,
|
|
21
|
+
onPageShow: callbacks.onPageShow,
|
|
22
|
+
onVisible: callbacks.onVisible
|
|
23
|
+
});
|
|
24
|
+
if (callbacks.onDeviceChange) {
|
|
25
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
26
|
+
"devicechange",
|
|
27
|
+
callbacks.onDeviceChange
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
return () => {
|
|
31
|
+
teardownBrowserLifecycle();
|
|
32
|
+
if (callbacks.onDeviceChange) {
|
|
33
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
34
|
+
"devicechange",
|
|
35
|
+
callbacks.onDeviceChange
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
async function replaceMicrophoneTrack(state) {
|
|
41
|
+
if (!state.peerConnection) {
|
|
42
|
+
throw new Error("Realtime transport not active");
|
|
43
|
+
}
|
|
44
|
+
const nextStream = await acquireMicrophoneStream();
|
|
45
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
46
|
+
if (!nextTrack) {
|
|
47
|
+
throw new Error("Microphone access required for Realtime API");
|
|
48
|
+
}
|
|
49
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
50
|
+
if (!sender) {
|
|
51
|
+
throw new Error("No outbound audio sender available");
|
|
52
|
+
}
|
|
53
|
+
await sender.replaceTrack(nextTrack);
|
|
54
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
55
|
+
return {
|
|
56
|
+
audioSender: sender,
|
|
57
|
+
mediaStream: nextStream
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
61
|
+
let pending = null;
|
|
62
|
+
return {
|
|
63
|
+
schedule() {
|
|
64
|
+
if (pending) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
pending = setTimeout(() => {
|
|
68
|
+
pending = null;
|
|
69
|
+
onDisconnect();
|
|
70
|
+
}, delayMs);
|
|
71
|
+
},
|
|
72
|
+
cancel() {
|
|
73
|
+
if (!pending) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
clearTimeout(pending);
|
|
77
|
+
pending = null;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// src/openai-agents/lip-sync-analyzer.ts
|
|
83
|
+
var LipSyncAnalyzer = class {
|
|
84
|
+
constructor(options) {
|
|
85
|
+
this.options = options;
|
|
86
|
+
}
|
|
87
|
+
audioAnalysisStream = null;
|
|
88
|
+
audioContext = null;
|
|
89
|
+
audioSource = null;
|
|
90
|
+
analyser = null;
|
|
91
|
+
lipSyncInterval = null;
|
|
92
|
+
pendingAudioElementPoll = null;
|
|
93
|
+
observedAudioElement = null;
|
|
94
|
+
observedAudioElementListener = null;
|
|
95
|
+
async prepareAudioContext() {
|
|
96
|
+
if (this.audioContext) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
100
|
+
if (!audioContextConstructor) {
|
|
101
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
102
|
+
}
|
|
103
|
+
this.audioContext = new audioContextConstructor();
|
|
104
|
+
}
|
|
105
|
+
observeAudioElement(audioElement) {
|
|
106
|
+
this.detachObservedAudioElement();
|
|
107
|
+
const tryAttachStream = () => {
|
|
108
|
+
const stream = audioElement.srcObject;
|
|
109
|
+
if (stream instanceof MediaStream) {
|
|
110
|
+
this.attachStream(stream);
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
this.observedAudioElement = audioElement;
|
|
114
|
+
this.observedAudioElementListener = tryAttachStream;
|
|
115
|
+
audioElement.addEventListener("loadedmetadata", tryAttachStream);
|
|
116
|
+
this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
|
|
117
|
+
}
|
|
118
|
+
attachStream(stream) {
|
|
119
|
+
if (this.audioAnalysisStream === stream) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
this.stopAudioElementPolling();
|
|
123
|
+
this.stopLipSyncAnalysis();
|
|
124
|
+
this.audioAnalysisStream = stream;
|
|
125
|
+
try {
|
|
126
|
+
if (!this.audioContext) {
|
|
127
|
+
void this.prepareAudioContext().catch((error) => {
|
|
128
|
+
this.options.onError?.(error);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
if (!this.audioContext) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
this.audioSource?.disconnect?.();
|
|
135
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
136
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
137
|
+
this.analyser.fftSize = 256;
|
|
138
|
+
this.analyser.smoothingTimeConstant = 0.8;
|
|
139
|
+
this.audioSource.connect(this.analyser);
|
|
140
|
+
this.startLipSyncAnalysis();
|
|
141
|
+
} catch (error) {
|
|
142
|
+
this.options.onError?.(error);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
stopOutput() {
|
|
146
|
+
this.stopLipSyncAnalysis();
|
|
147
|
+
}
|
|
148
|
+
pause() {
|
|
149
|
+
this.stopLipSyncAnalysis();
|
|
150
|
+
}
|
|
151
|
+
resume() {
|
|
152
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
this.startLipSyncAnalysis();
|
|
156
|
+
}
|
|
157
|
+
cleanup() {
|
|
158
|
+
this.stopLipSyncAnalysis();
|
|
159
|
+
this.stopAudioElementPolling();
|
|
160
|
+
this.detachObservedAudioElement();
|
|
161
|
+
this.audioSource?.disconnect?.();
|
|
162
|
+
this.audioSource = null;
|
|
163
|
+
if (this.audioContext) {
|
|
164
|
+
void this.audioContext.close();
|
|
165
|
+
this.audioContext = null;
|
|
166
|
+
}
|
|
167
|
+
this.audioAnalysisStream = null;
|
|
168
|
+
this.analyser = null;
|
|
169
|
+
}
|
|
170
|
+
startLipSyncAnalysis() {
|
|
171
|
+
if (!this.analyser) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
175
|
+
const dataArray = new Uint8Array(bufferLength);
|
|
176
|
+
this.lipSyncInterval = window.setInterval(() => {
|
|
177
|
+
if (!this.analyser) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
this.analyser.getByteFrequencyData(dataArray);
|
|
181
|
+
let sum = 0;
|
|
182
|
+
for (let index = 0; index < bufferLength; index += 1) {
|
|
183
|
+
const normalized = dataArray[index] / 255;
|
|
184
|
+
sum += normalized * normalized;
|
|
185
|
+
}
|
|
186
|
+
const rms = Math.sqrt(sum / bufferLength);
|
|
187
|
+
this.options.onRms(Math.min(rms * 3, 1));
|
|
188
|
+
}, 1e3 / 60);
|
|
189
|
+
}
|
|
190
|
+
stopLipSyncAnalysis() {
|
|
191
|
+
if (this.lipSyncInterval) {
|
|
192
|
+
clearInterval(this.lipSyncInterval);
|
|
193
|
+
this.lipSyncInterval = null;
|
|
194
|
+
}
|
|
195
|
+
this.options.onRms(0);
|
|
196
|
+
}
|
|
197
|
+
stopAudioElementPolling() {
|
|
198
|
+
if (this.pendingAudioElementPoll) {
|
|
199
|
+
clearInterval(this.pendingAudioElementPoll);
|
|
200
|
+
this.pendingAudioElementPoll = null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
detachObservedAudioElement() {
|
|
204
|
+
if (this.observedAudioElement && this.observedAudioElementListener) {
|
|
205
|
+
this.observedAudioElement.removeEventListener(
|
|
206
|
+
"loadedmetadata",
|
|
207
|
+
this.observedAudioElementListener
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
this.observedAudioElement = null;
|
|
211
|
+
this.observedAudioElementListener = null;
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
// src/openai/defaults.ts
|
|
216
|
+
var DEFAULT_OPENAI_REALTIME_MODEL = "gpt-realtime-mini";
|
|
217
|
+
var DEFAULT_OPENAI_REALTIME_VOICE = "marin";
|
|
218
|
+
var DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
219
|
+
|
|
220
|
+
export { DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS, DEFAULT_OPENAI_REALTIME_MODEL, DEFAULT_OPENAI_REALTIME_VOICE, LipSyncAnalyzer, acquireMicrophoneStream, bindTransportLifecycle, createIceDisconnectDebouncer, replaceMicrophoneTrack };
|
|
221
|
+
//# sourceMappingURL=chunk-3PIMXNQU.mjs.map
|
|
222
|
+
//# sourceMappingURL=chunk-3PIMXNQU.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/internal/microphone.ts","../src/internal/webrtc-lifecycle.ts","../src/openai-agents/lip-sync-analyzer.ts","../src/openai/defaults.ts"],"names":[],"mappings":";;;;AAAO,IAAM,8BAAA,GAAiC;AAAA,EAC5C,gBAAA,EAAkB,IAAA;AAAA,EAClB,gBAAA,EAAkB,IAAA;AAAA,EAClB,eAAA,EAAiB;AACnB,CAAA;AAEA,eAAsB,uBAAA,GAAgD;AACpE,EAAA,OAAO,SAAA,CAAU,aAAa,YAAA,CAAa;AAAA,IACzC,KAAA,EAAO;AAAA,GACR,CAAA;AACH;ACPO,IAAM,oCAAA,GAAuC;AAW7C,SAAS,uBACd,SAAA,EACY;AACZ,EAAA,MAAM,2BAA2B,yBAAA,CAA0B;AAAA,IACzD,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,WAAW,SAAA,CAAU;AAAA,GACtB,CAAA;AAED,EAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,IAAA,SAAA,CAAU,YAAA,EAAc,gBAAA;AAAA,MACtB,cAAA;AAAA,MACA,SAAA,CAAU;AAAA,KACZ;AAAA,EACF;AAEA,EAAA,OAAO,MAAM;AACX,IAAA,wBAAA,EAAyB;AACzB,IAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,MAAA,SAAA,CAAU,YAAA,EAAc,mBAAA;AAAA,QACtB,cAAA;AAAA,QACA,SAAA,CAAU;AAAA,OACZ;AAAA,IACF;AAAA,EACF,CAAA;AACF;AAQA,eAAsB,uBACpB,KAAA,EACkE;AAClE,EAAA,IAAI,CAAC,MAAM,cAAA,EAAgB;AACzB,IAAA,MAAM,IAAI,MAAM,+BAA+B,CAAA;AAAA,EACjD;AAEA,EAAA,MAAM,UAAA,GAAa,MAAM,uBAAA,EAAwB;AACjD,EAAA,MAAM,SAAA,GAAY,UAAA,CAAW,cAAA,EAAe,CAAE,CAAC,CAAA;AAC/C,EAAA,IAAI,CAAC,SAAA,EAAW;AACd,IAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,EAC/D;AAEA,EAAA,MAAM,MAAA,GACJ,MAAM,WAAA,KACL,OAAO,MAAM,cAAA,CAAe,UAAA,KAAe,aACvC,KAAA,CAAM,cAAA,CACJ,YAAW,CACX,IAAA,CAAK,CAAC,SAAA,KAAc,SAAA,CAAU,OAAO,IAAA,KAAS,OAAO,KAAK,IAAA,GAC7D,IAAA,CAAA;AACN,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,EACtD;AAEA,EAAA,MAAM,MAAA,CAAO,aAAa,SAAS,CAAA;AACnC,EAAA,KAAA,CAAM,WAAA,EAAa,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAE9D,EAAA,OAAO;AAAA,IACL,WAAA,EAAa,MAAA;AAAA,IACb,WAAA,EAAa;AAAA,GACf;AACF;AAEO,SAAS,4BAAA,CACd,YAAA,EACA,OAAA,GAAU,oCAAA,EAIV;AACA,EAAA,IAAI,OAAA,GAAgD,IAAA;AAEpD,EAAA,OAAO;AAAA,IACL,QAAA,GAAW;AACT,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AAEA,MAAA,OAAA,GAAU,WAAW,MAAM;AACzB,QAAA,OAAA,GAAU,IAAA;AACV,QAAA,YAAA,EAAa;AAAA,MACf,GAAG,OAAO,CAAA;AAAA,IACZ,CAAA;AAAA,IACA,MAAA,GAAS;AACP,MAAA,IAAI,CAAC,OAAA,EAAS;AACZ,QAAA;AAAA,MACF;AAEA,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,OAAA,GAAU,IAAA;AAAA,IACZ;AAAA,GACF;AACF;;;AC1GO,IAAM,kBAAN,MAAsB;AAAA,EAU3B,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAAkC;AAAA,EAT9C,mBAAA,GAA0C,IAAA;AAAA,EAC1C,YAAA,GAAoC,IAAA;AAAA,EACpC,WAAA,GAAiD,IAAA;AAAA,EACjD,QAAA,GAAgC,IAAA;AAAA,EAChC,eAAA,GAAiC,IAAA;AAAA,EACjC,uBAAA,GAAyC,IAAA;AAAA,EACzC,oBAAA,GAAgD,IAAA;AAAA,EAChD,4BAAA,GAAoD,IAAA;AAAA,EAI5D,MAAM,mBAAA,GAAqC;AACzC,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GACJ,MAAA,CAAO,YAAA,IACN,MAAA,CACE,kBAAA;AACL,IAAA,IAAI,CAAC,uBAAA,EAAyB;AAC5B,MAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AAAA,IACjE;AAEA,IAAA,IAAA,CAAK,YAAA,GAAe,IAAI,uBAAA,EAAwB;AAAA,EAClD;AAAA,EAEA,oBAAoB,YAAA,EAAsC;AACxD,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,MAAM,kBAAkB,MAAM;AAC5B,MAAA,MAAM,SAAS,YAAA,CAAa,SAAA;AAC5B,MAAA,IAAI,kBAAkB,WAAA,EAAa;AACjC,QAAA,IAAA,CAAK,aAAa,MAAM,CAAA;AAAA,MAC1B;AAAA,IACF,CAAA;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,YAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,eAAA;AACpC,IAAA,YAAA,CAAa,gBAAA,CAAiB,kBAAkB,eAAe,CAAA;AAC/D,IAAA,IAAA,CAAK,uBAAA,GAA0B,MAAA,CAAO,WAAA,CAAY,eAAA,EAAiB,EAAE,CAAA;AAAA,EACvE;AAAA,EAEA,aAAa,MAAA,EAA2B;AACtC,IAAA,IAAI,IAAA,CAAK,wBAAwB,MAAA,EAAQ;AACvC,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,mBAAA,GAAsB,MAAA;AAE3B,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA,KAAK,IAAA,CAAK,mBAAA,EAAoB,CAAE,KAAA,CAAM,CAAC,KAAA,KAAU;AAC/C,UAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,QAC9B,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA,CAAK,YAAA,CAAa,uBAAA,CAAwB,MAAM,CAAA;AACnE,MAAA,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,YAAA,CAAa,cAAA,EAAe;AACjD,MAAA,IAAA,CAAK,SAAS,OAAA,GAAU,GAAA;AACxB,MAAA,IAAA,CAAK,SAAS,qBAAA,GAAwB,GAAA;AACtC,MAAA,IAAA,CAAK,WAAA,CAAY,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AAEtC,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,IAC5B,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,UAAA,GAAmB;AACjB,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,KAAA,GAAc;AACZ,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,MAAA,GAAe;AACb,IAAA,IAAI,CAAC,IAAA,CAAK,QAAA,IAAY,IAAA,CAAK,eAAA,EAAiB;AAC1C,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,OAAA,GAAgB;AACd,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAEnB,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,KAAK,IAAA,CAAK,aAAa,KAAA,EAAM;AAC7B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,mBAAA,GAAsB,IAAA;AAC3B,IAAA,IAAA,CAAK,QAAA,GAAW,IAAA;AAAA,EAClB;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,QAAA,CAAS,iBAAA;AACnC,IAAA,MAAM,SAAA,GAAY,IAAI,UAAA,CAAW,YAAY,CAAA;AAE7C,IAAA,IAAA,CAAK,eAAA,GAAkB,MAAA,CAAO,WAAA,CAAY,MAAM;AAC9C,MAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,QAAA,CAAS,qBAAqB,SAAS,CAAA;AAE5C,MAAA,IAAI,GAAA,GAAM,CAAA;AACV,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,YAAA,EAAc,SAAS,CAAA,EAAG;AACpD,QAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAK,CAAA,GAAI,GAAA;AACtC,QAAA,GAAA,IAAO,UAAA,GAAa,UAAA;AAAA,MACtB;AAEA,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,IAAA,CAAK,GAAA,GAAM,YAAY,CAAA;AACxC,MAAA,IAAA,CAAK,QAAQ,KAAA,CAAM,IAAA,CAAK,IAAI,GAAA,GAAM,CAAA,EAAG,CAAC,CAAC,CAAA;AAAA,IACzC,CAAA,EAAG,MAAO,EAAE,CAAA;AAAA,EACd;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,aAAA,CAAc,KAAK,eAAe,CAAA;AAClC,MAAA,IAAA,CAAK,eAAA,GAAkB,IAAA;AAAA,IACzB;AACA,IAAA,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAC,CAAA;AAAA,EACtB;AAAA,EAEQ,uBAAA,GAAgC;AACtC,IAAA,IAAI,KAAK,uBAAA,EAAyB;AAChC,MAAA,aAAA,CAAc,KAAK,uBAAuB,CAAA;AAC1C,MAAA,IAAA,CAAK,uBAAA,GAA0B,IAAA;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,0BAAA,GAAmC;AACzC,IAAA,IAAI,IAAA,CAAK,oBAAA,IAAwB,IAAA,CAAK,4BAAA,EAA8B;AAClE,MAAA,IAAA,CAAK,oBAAA,CAAqB,mBAAA;AAAA,QACxB,gBAAA;AAAA,QACA,IAAA,CAAK;AAAA,OACP;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,IAAA;AAAA,EACtC;AACF;;;ACnKO,IAAM,6BAAA,GAAgC;AACtC,IAAM,6BAAA,GAAgC;AACtC,IAAM,0CAAA,GACX","file":"chunk-3PIMXNQU.mjs","sourcesContent":["export const DEFAULT_MICROPHONE_CONSTRAINTS = {\n echoCancellation: true,\n noiseSuppression: true,\n autoGainControl: true,\n} satisfies MediaTrackConstraints;\n\nexport async function acquireMicrophoneStream(): Promise<MediaStream> {\n return navigator.mediaDevices.getUserMedia({\n audio: DEFAULT_MICROPHONE_CONSTRAINTS,\n });\n}\n","import { subscribeBrowserLifecycle } from \"@charivo/core\";\nimport { acquireMicrophoneStream } from \"./microphone\";\n\nexport const DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1_000;\n\nexport interface TransportLifecycleCallbacks {\n onHidden?(): void;\n onOnline?(): void;\n onPageHide?(): void;\n onPageShow?(event: PageTransitionEvent): void;\n onVisible?(): void;\n onDeviceChange?(): void;\n}\n\nexport function bindTransportLifecycle(\n callbacks: TransportLifecycleCallbacks,\n): () => void {\n const teardownBrowserLifecycle = subscribeBrowserLifecycle({\n onHidden: callbacks.onHidden,\n onOnline: callbacks.onOnline,\n onPageHide: callbacks.onPageHide,\n onPageShow: callbacks.onPageShow,\n onVisible: callbacks.onVisible,\n });\n\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.addEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n\n return () => {\n teardownBrowserLifecycle();\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.removeEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n };\n}\n\nexport interface AudioSenderTrackState {\n audioSender: RTCRtpSender | null;\n mediaStream: MediaStream | null;\n peerConnection: RTCPeerConnection | null;\n}\n\nexport async function replaceMicrophoneTrack(\n state: AudioSenderTrackState,\n): Promise<{ audioSender: RTCRtpSender; mediaStream: MediaStream }> {\n if (!state.peerConnection) {\n throw new Error(\"Realtime transport not active\");\n }\n\n const nextStream = await acquireMicrophoneStream();\n const nextTrack = nextStream.getAudioTracks()[0];\n if (!nextTrack) {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n const sender =\n state.audioSender ??\n (typeof state.peerConnection.getSenders === \"function\"\n ? (state.peerConnection\n .getSenders()\n .find((candidate) => candidate.track?.kind === \"audio\") ?? null)\n : null);\n if (!sender) {\n throw new Error(\"No outbound audio sender available\");\n }\n\n await sender.replaceTrack(nextTrack);\n state.mediaStream?.getTracks().forEach((track) => track.stop());\n\n return {\n audioSender: sender,\n mediaStream: nextStream,\n };\n}\n\nexport function createIceDisconnectDebouncer(\n onDisconnect: () => void,\n delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS,\n): {\n schedule(): void;\n cancel(): void;\n} {\n let pending: ReturnType<typeof setTimeout> | null = null;\n\n return {\n schedule() {\n if (pending) {\n return;\n }\n\n pending = setTimeout(() => {\n pending = null;\n onDisconnect();\n }, delayMs);\n },\n cancel() {\n if (!pending) {\n return;\n }\n\n clearTimeout(pending);\n pending = null;\n },\n };\n}\n","interface LipSyncAnalyzerOptions {\n onRms: (rms: number) => void;\n onError?: (error: unknown) => void;\n}\n\nexport class LipSyncAnalyzer {\n private audioAnalysisStream: MediaStream | null = null;\n private audioContext: AudioContext | null = null;\n private audioSource: MediaStreamAudioSourceNode | null = null;\n private analyser: AnalyserNode | null = null;\n private lipSyncInterval: number | null = null;\n private pendingAudioElementPoll: number | null = null;\n private observedAudioElement: HTMLAudioElement | null = null;\n private observedAudioElementListener: (() => void) | null = null;\n\n constructor(private options: LipSyncAnalyzerOptions) {}\n\n async prepareAudioContext(): Promise<void> {\n if (this.audioContext) {\n return;\n }\n\n const audioContextConstructor =\n window.AudioContext ||\n (window as Window & { webkitAudioContext?: typeof AudioContext })\n .webkitAudioContext;\n if (!audioContextConstructor) {\n throw new Error(\"AudioContext is not supported in this browser\");\n }\n\n this.audioContext = new audioContextConstructor();\n }\n\n observeAudioElement(audioElement: HTMLAudioElement): void {\n this.detachObservedAudioElement();\n\n const tryAttachStream = () => {\n const stream = audioElement.srcObject;\n if (stream instanceof MediaStream) {\n this.attachStream(stream);\n }\n };\n\n this.observedAudioElement = audioElement;\n this.observedAudioElementListener = tryAttachStream;\n audioElement.addEventListener(\"loadedmetadata\", tryAttachStream);\n this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);\n }\n\n attachStream(stream: MediaStream): void {\n if (this.audioAnalysisStream === stream) {\n return;\n }\n\n this.stopAudioElementPolling();\n this.stopLipSyncAnalysis();\n this.audioAnalysisStream = stream;\n\n try {\n if (!this.audioContext) {\n void this.prepareAudioContext().catch((error) => {\n this.options.onError?.(error);\n });\n }\n\n if (!this.audioContext) {\n return;\n }\n\n this.audioSource?.disconnect?.();\n this.audioSource = this.audioContext.createMediaStreamSource(stream);\n this.analyser = this.audioContext.createAnalyser();\n this.analyser.fftSize = 256;\n this.analyser.smoothingTimeConstant = 0.8;\n this.audioSource.connect(this.analyser);\n\n this.startLipSyncAnalysis();\n } catch (error) {\n this.options.onError?.(error);\n }\n }\n\n stopOutput(): void {\n this.stopLipSyncAnalysis();\n }\n\n pause(): void {\n this.stopLipSyncAnalysis();\n }\n\n resume(): void {\n if (!this.analyser || this.lipSyncInterval) {\n return;\n }\n\n this.startLipSyncAnalysis();\n }\n\n cleanup(): void {\n this.stopLipSyncAnalysis();\n this.stopAudioElementPolling();\n this.detachObservedAudioElement();\n\n this.audioSource?.disconnect?.();\n this.audioSource = null;\n\n if (this.audioContext) {\n void this.audioContext.close();\n this.audioContext = null;\n }\n\n this.audioAnalysisStream = null;\n this.analyser = null;\n }\n\n private startLipSyncAnalysis(): void {\n if (!this.analyser) {\n return;\n }\n\n const bufferLength = this.analyser.frequencyBinCount;\n const dataArray = new Uint8Array(bufferLength);\n\n this.lipSyncInterval = window.setInterval(() => {\n if (!this.analyser) {\n return;\n }\n\n this.analyser.getByteFrequencyData(dataArray);\n\n let sum = 0;\n for (let index = 0; index < bufferLength; index += 1) {\n const normalized = dataArray[index] / 255;\n sum += normalized * normalized;\n }\n\n const rms = Math.sqrt(sum / bufferLength);\n this.options.onRms(Math.min(rms * 3, 1));\n }, 1000 / 60);\n }\n\n private stopLipSyncAnalysis(): void {\n if (this.lipSyncInterval) {\n clearInterval(this.lipSyncInterval);\n this.lipSyncInterval = null;\n }\n this.options.onRms(0);\n }\n\n private stopAudioElementPolling(): void {\n if (this.pendingAudioElementPoll) {\n clearInterval(this.pendingAudioElementPoll);\n this.pendingAudioElementPoll = null;\n }\n }\n\n private detachObservedAudioElement(): void {\n if (this.observedAudioElement && this.observedAudioElementListener) {\n this.observedAudioElement.removeEventListener(\n \"loadedmetadata\",\n this.observedAudioElementListener,\n );\n }\n\n this.observedAudioElement = null;\n this.observedAudioElementListener = null;\n }\n}\n","import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS as DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS } from \"../instructions\";\n\n// Mirrors packages/server/src/openai/realtime/index.ts until the server/browser\n// OpenAI defaults consolidate under ROADMAP P0-ARCH-2.\nexport const DEFAULT_OPENAI_REALTIME_MODEL = \"gpt-realtime-mini\";\nexport const DEFAULT_OPENAI_REALTIME_VOICE = \"marin\";\nexport const DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS =\n DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS;\n"]}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { delay } from './chunk-XPKFJWHL.mjs';
|
|
2
|
+
import { LipSyncAnalyzer, createIceDisconnectDebouncer, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, acquireMicrophoneStream, bindTransportLifecycle, replaceMicrophoneTrack, DEFAULT_OPENAI_REALTIME_VOICE } from './chunk-3PIMXNQU.mjs';
|
|
3
|
+
import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-PNBYFAER.mjs';
|
|
3
4
|
import { OPENAI_REALTIME_ADAPTER } from '@charivo/core';
|
|
4
5
|
|
|
5
6
|
var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
@@ -21,10 +22,10 @@ var OpenAIRealtimeClient = class {
|
|
|
21
22
|
audioElement = null;
|
|
22
23
|
mediaStream = null;
|
|
23
24
|
audioSender = null;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
26
|
+
onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
|
|
27
|
+
onError: (error) => console.error("Failed to setup audio analysis:", error)
|
|
28
|
+
});
|
|
28
29
|
isResponseInProgress = false;
|
|
29
30
|
cancelInFlight = false;
|
|
30
31
|
hasStartedAssistantResponse = false;
|
|
@@ -56,14 +57,14 @@ var OpenAIRealtimeClient = class {
|
|
|
56
57
|
this.audioElement = document.createElement("audio");
|
|
57
58
|
this.audioElement.autoplay = true;
|
|
58
59
|
this.audioElement.setAttribute("playsinline", "true");
|
|
59
|
-
this.
|
|
60
|
+
void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
|
|
60
61
|
this.bindConnectionEvents();
|
|
61
62
|
this.bindTransportLifecycleEvents();
|
|
62
63
|
this.pc.ontrack = (event) => {
|
|
63
64
|
if (this.audioElement) {
|
|
64
65
|
this.audioElement.srcObject = event.streams[0];
|
|
65
66
|
}
|
|
66
|
-
this.
|
|
67
|
+
this.lipSyncAnalyzer.attachStream(event.streams[0]);
|
|
67
68
|
};
|
|
68
69
|
try {
|
|
69
70
|
this.mediaStream = await acquireMicrophoneStream();
|
|
@@ -90,14 +91,10 @@ var OpenAIRealtimeClient = class {
|
|
|
90
91
|
session: config ?? {},
|
|
91
92
|
sdpOffer: offer.sdp
|
|
92
93
|
});
|
|
93
|
-
|
|
94
|
-
throw new Error(
|
|
95
|
-
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
96
|
-
);
|
|
97
|
-
}
|
|
94
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
98
95
|
await this.pc.setRemoteDescription({
|
|
99
96
|
type: "answer",
|
|
100
|
-
sdp:
|
|
97
|
+
sdp: answerSdp
|
|
101
98
|
});
|
|
102
99
|
this.emitEvent({ type: "session.started" });
|
|
103
100
|
} catch (error) {
|
|
@@ -286,10 +283,7 @@ var OpenAIRealtimeClient = class {
|
|
|
286
283
|
if (this.cancelInFlight || !event.delta) {
|
|
287
284
|
return;
|
|
288
285
|
}
|
|
289
|
-
|
|
290
|
-
this.hasStartedAssistantResponse = true;
|
|
291
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
292
|
-
}
|
|
286
|
+
this.ensureAssistantResponseStarted();
|
|
293
287
|
this.assistantText += event.delta;
|
|
294
288
|
this.emitEvent({
|
|
295
289
|
type: "assistant.text.delta",
|
|
@@ -300,39 +294,13 @@ var OpenAIRealtimeClient = class {
|
|
|
300
294
|
if (this.cancelInFlight || !event.text) {
|
|
301
295
|
return;
|
|
302
296
|
}
|
|
303
|
-
|
|
304
|
-
this.hasStartedAssistantResponse = true;
|
|
305
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
306
|
-
}
|
|
307
|
-
if (event.text !== this.assistantText) {
|
|
308
|
-
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
309
|
-
this.assistantText = event.text;
|
|
310
|
-
if (delta) {
|
|
311
|
-
this.emitEvent({
|
|
312
|
-
type: "assistant.text.delta",
|
|
313
|
-
text: delta
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
}
|
|
297
|
+
this.emitFinalAssistantText(event.text);
|
|
317
298
|
return;
|
|
318
299
|
case "response.output_audio_transcript.done":
|
|
319
300
|
if (this.cancelInFlight || !event.transcript) {
|
|
320
301
|
return;
|
|
321
302
|
}
|
|
322
|
-
|
|
323
|
-
this.hasStartedAssistantResponse = true;
|
|
324
|
-
this.emitEvent({ type: "assistant.response.started" });
|
|
325
|
-
}
|
|
326
|
-
if (event.transcript !== this.assistantText) {
|
|
327
|
-
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
328
|
-
this.assistantText = event.transcript;
|
|
329
|
-
if (delta) {
|
|
330
|
-
this.emitEvent({
|
|
331
|
-
type: "assistant.text.delta",
|
|
332
|
-
text: delta
|
|
333
|
-
});
|
|
334
|
-
}
|
|
335
|
-
}
|
|
303
|
+
this.emitFinalAssistantText(event.transcript);
|
|
336
304
|
return;
|
|
337
305
|
case "conversation.item.input_audio_transcription.completed":
|
|
338
306
|
if (event.transcript) {
|
|
@@ -463,17 +431,17 @@ var OpenAIRealtimeClient = class {
|
|
|
463
431
|
this.emitConnectionLost("online");
|
|
464
432
|
};
|
|
465
433
|
handleHidden = () => {
|
|
466
|
-
this.
|
|
434
|
+
this.lipSyncAnalyzer.pause();
|
|
467
435
|
};
|
|
468
436
|
handleVisible = () => {
|
|
469
|
-
this.
|
|
437
|
+
this.lipSyncAnalyzer.resume();
|
|
470
438
|
this.emitConnectionLost("visibility");
|
|
471
439
|
};
|
|
472
440
|
handlePageHide = () => {
|
|
473
|
-
this.
|
|
441
|
+
this.lipSyncAnalyzer.pause();
|
|
474
442
|
};
|
|
475
443
|
handlePageShow = (event) => {
|
|
476
|
-
this.
|
|
444
|
+
this.lipSyncAnalyzer.resume();
|
|
477
445
|
if (event.persisted) {
|
|
478
446
|
this.emitConnectionLost("pageshow");
|
|
479
447
|
}
|
|
@@ -508,7 +476,7 @@ var OpenAIRealtimeClient = class {
|
|
|
508
476
|
this.isResponseInProgress = false;
|
|
509
477
|
this.cancelInFlight = false;
|
|
510
478
|
this.resetResponseTracking();
|
|
511
|
-
this.
|
|
479
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
512
480
|
this.emitEvent({
|
|
513
481
|
type: "connection.lost",
|
|
514
482
|
cause,
|
|
@@ -524,77 +492,11 @@ var OpenAIRealtimeClient = class {
|
|
|
524
492
|
}
|
|
525
493
|
return true;
|
|
526
494
|
}
|
|
527
|
-
prepareAudioAnalysis() {
|
|
528
|
-
if (this.audioContext) {
|
|
529
|
-
return;
|
|
530
|
-
}
|
|
531
|
-
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
532
|
-
if (!audioContextConstructor) {
|
|
533
|
-
return;
|
|
534
|
-
}
|
|
535
|
-
this.audioContext = new audioContextConstructor();
|
|
536
|
-
}
|
|
537
|
-
setupAudioAnalysis(stream) {
|
|
538
|
-
try {
|
|
539
|
-
this.prepareAudioAnalysis();
|
|
540
|
-
if (!this.audioContext) {
|
|
541
|
-
return;
|
|
542
|
-
}
|
|
543
|
-
this.audioSource?.disconnect();
|
|
544
|
-
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
545
|
-
this.analyser = this.audioContext.createAnalyser();
|
|
546
|
-
this.analyser.fftSize = 256;
|
|
547
|
-
this.analyser.smoothingTimeConstant = 0.8;
|
|
548
|
-
this.audioSource.connect(this.analyser);
|
|
549
|
-
this.startLipSyncAnalysis();
|
|
550
|
-
} catch (error) {
|
|
551
|
-
console.error("Failed to setup audio analysis:", error);
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
startLipSyncAnalysis() {
|
|
555
|
-
if (!this.analyser) {
|
|
556
|
-
return;
|
|
557
|
-
}
|
|
558
|
-
const bufferLength = this.analyser.frequencyBinCount;
|
|
559
|
-
const dataArray = new Uint8Array(bufferLength);
|
|
560
|
-
this.lipSyncInterval = window.setInterval(() => {
|
|
561
|
-
if (!this.analyser) {
|
|
562
|
-
return;
|
|
563
|
-
}
|
|
564
|
-
this.analyser.getByteFrequencyData(dataArray);
|
|
565
|
-
let sum = 0;
|
|
566
|
-
for (let index = 0; index < bufferLength; index += 1) {
|
|
567
|
-
const normalized = dataArray[index] / 255;
|
|
568
|
-
sum += normalized * normalized;
|
|
569
|
-
}
|
|
570
|
-
const rms = Math.sqrt(sum / bufferLength);
|
|
571
|
-
this.emitEvent({
|
|
572
|
-
type: "audio.lipsync",
|
|
573
|
-
rms: Math.min(rms * 3, 1)
|
|
574
|
-
});
|
|
575
|
-
}, 1e3 / 60);
|
|
576
|
-
}
|
|
577
|
-
stopLipSyncAnalysis() {
|
|
578
|
-
if (this.lipSyncInterval) {
|
|
579
|
-
clearInterval(this.lipSyncInterval);
|
|
580
|
-
this.lipSyncInterval = null;
|
|
581
|
-
}
|
|
582
|
-
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
583
|
-
}
|
|
584
|
-
pauseAudioAnalysis() {
|
|
585
|
-
this.stopLipSyncAnalysis();
|
|
586
|
-
}
|
|
587
|
-
resumeAudioAnalysis() {
|
|
588
|
-
if (!this.analyser || this.lipSyncInterval) {
|
|
589
|
-
return;
|
|
590
|
-
}
|
|
591
|
-
this.startLipSyncAnalysis();
|
|
592
|
-
}
|
|
593
495
|
cleanup() {
|
|
594
496
|
this.isCleaningUp = true;
|
|
595
497
|
this.unbindTransportLifecycleEvents();
|
|
596
498
|
this.iceDisconnectDebouncer.cancel();
|
|
597
|
-
this.
|
|
499
|
+
this.lipSyncAnalyzer.cleanup();
|
|
598
500
|
this.rejectPendingSessionUpdate(
|
|
599
501
|
new Error("Realtime session ended before session update completed")
|
|
600
502
|
);
|
|
@@ -614,13 +516,7 @@ var OpenAIRealtimeClient = class {
|
|
|
614
516
|
this.audioElement.srcObject = null;
|
|
615
517
|
this.audioElement = null;
|
|
616
518
|
}
|
|
617
|
-
if (this.audioContext) {
|
|
618
|
-
this.audioContext.close();
|
|
619
|
-
this.audioContext = null;
|
|
620
|
-
}
|
|
621
|
-
this.audioSource = null;
|
|
622
519
|
this.audioSender = null;
|
|
623
|
-
this.analyser = null;
|
|
624
520
|
this.isResponseInProgress = false;
|
|
625
521
|
this.cancelInFlight = false;
|
|
626
522
|
this.connectionLossNotified = false;
|
|
@@ -668,14 +564,10 @@ var OpenAIRealtimeClient = class {
|
|
|
668
564
|
session: config ?? this.currentSessionConfig ?? {},
|
|
669
565
|
sdpOffer: offer.sdp
|
|
670
566
|
});
|
|
671
|
-
|
|
672
|
-
throw new Error(
|
|
673
|
-
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
674
|
-
);
|
|
675
|
-
}
|
|
567
|
+
const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
|
|
676
568
|
await pc.setRemoteDescription({
|
|
677
569
|
type: "answer",
|
|
678
|
-
sdp:
|
|
570
|
+
sdp: answerSdp
|
|
679
571
|
});
|
|
680
572
|
return true;
|
|
681
573
|
}
|
|
@@ -713,11 +605,38 @@ var OpenAIRealtimeClient = class {
|
|
|
713
605
|
getSessionUpdateTimeoutMs() {
|
|
714
606
|
return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
|
|
715
607
|
}
|
|
608
|
+
resolveWebRTCAnswerSdp(bootstrap) {
|
|
609
|
+
if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
610
|
+
throw new Error(
|
|
611
|
+
`OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
return bootstrap.answerSdp;
|
|
615
|
+
}
|
|
716
616
|
resetResponseTracking() {
|
|
717
617
|
this.assistantText = "";
|
|
718
618
|
this.hasStartedAssistantResponse = false;
|
|
719
619
|
this.hasStartedAudioOutput = false;
|
|
720
620
|
}
|
|
621
|
+
ensureAssistantResponseStarted() {
|
|
622
|
+
if (!this.hasStartedAssistantResponse) {
|
|
623
|
+
this.hasStartedAssistantResponse = true;
|
|
624
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
emitFinalAssistantText(full) {
|
|
628
|
+
this.ensureAssistantResponseStarted();
|
|
629
|
+
if (full !== this.assistantText) {
|
|
630
|
+
const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
|
|
631
|
+
this.assistantText = full;
|
|
632
|
+
if (delta) {
|
|
633
|
+
this.emitEvent({
|
|
634
|
+
type: "assistant.text.delta",
|
|
635
|
+
text: delta
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
721
640
|
emitEvent(event) {
|
|
722
641
|
for (const callback of this.eventCallbacks) {
|
|
723
642
|
callback(event);
|
|
@@ -739,12 +658,21 @@ function createOpenAIRealtimeClient(options) {
|
|
|
739
658
|
return new OpenAIRealtimeClient(options);
|
|
740
659
|
}
|
|
741
660
|
function toOpenAIRealtimeSessionUpdate(config) {
|
|
661
|
+
const audio = {
|
|
662
|
+
output: {
|
|
663
|
+
voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
|
|
664
|
+
}
|
|
665
|
+
};
|
|
666
|
+
const transcription = config?.inputAudioTranscription;
|
|
667
|
+
if (transcription !== void 0) {
|
|
668
|
+
if (transcription.enabled === false) {
|
|
669
|
+
audio.input = { transcription: null };
|
|
670
|
+
} else if (transcription.model !== void 0) {
|
|
671
|
+
audio.input = { transcription: { model: transcription.model } };
|
|
672
|
+
}
|
|
673
|
+
}
|
|
742
674
|
const session = {
|
|
743
|
-
audio
|
|
744
|
-
output: {
|
|
745
|
-
voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
|
|
746
|
-
}
|
|
747
|
-
},
|
|
675
|
+
audio,
|
|
748
676
|
tool_choice: config?.toolChoice ?? "auto"
|
|
749
677
|
};
|
|
750
678
|
if (config?.instructions) {
|
|
@@ -756,17 +684,12 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
756
684
|
if (config?.maxTokens !== void 0) {
|
|
757
685
|
session.max_response_output_tokens = config.maxTokens;
|
|
758
686
|
}
|
|
759
|
-
if (config
|
|
760
|
-
session.tools = config.tools;
|
|
687
|
+
if (config && "tools" in config) {
|
|
688
|
+
session.tools = config.tools ?? [];
|
|
761
689
|
}
|
|
762
690
|
return session;
|
|
763
691
|
}
|
|
764
|
-
function delay(ms) {
|
|
765
|
-
return new Promise((resolve) => {
|
|
766
|
-
setTimeout(resolve, ms);
|
|
767
|
-
});
|
|
768
|
-
}
|
|
769
692
|
|
|
770
693
|
export { OpenAIRealtimeClient, createOpenAIRealtimeClient };
|
|
771
|
-
//# sourceMappingURL=chunk-
|
|
772
|
-
//# sourceMappingURL=chunk-
|
|
694
|
+
//# sourceMappingURL=chunk-MI4QTRWT.mjs.map
|
|
695
|
+
//# sourceMappingURL=chunk-MI4QTRWT.mjs.map
|