@charivo/realtime 0.7.2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -50,10 +50,24 @@ await manager.startSession({
50
50
  provider: "openai",
51
51
  model: "gpt-realtime-mini",
52
52
  });
53
+ ```
54
+
55
+ `gpt-realtime-mini` is the default realtime model; the full `gpt-realtime` is available but meaningfully more expensive — consult [OpenAI's pricing page](https://openai.com/api/pricing/) before switching.
53
56
 
57
+ ```ts
54
58
  await manager.updateSession({
55
59
  voice: "alloy",
56
60
  });
61
+
62
+ // Switch the user-transcription model (cost vs. quality), or disable it.
63
+ await manager.updateSession({
64
+ inputAudioTranscription: { model: "gpt-4o-mini-transcribe" },
65
+ });
66
+
67
+ // Disable user transcription entirely (no whisper-1 charge):
68
+ await manager.updateSession({
69
+ inputAudioTranscription: { enabled: false },
70
+ });
57
71
  ```
58
72
 
59
73
  If the live transport drops temporarily, `RealtimeManager` now keeps the
@@ -0,0 +1,222 @@
1
+ import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS } from './chunk-PNBYFAER.mjs';
2
+ import { subscribeBrowserLifecycle } from '@charivo/core';
3
+
4
+ // src/internal/microphone.ts
5
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
6
+ echoCancellation: true,
7
+ noiseSuppression: true,
8
+ autoGainControl: true
9
+ };
10
+ async function acquireMicrophoneStream() {
11
+ return navigator.mediaDevices.getUserMedia({
12
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
13
+ });
14
+ }
15
+ var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
16
+ function bindTransportLifecycle(callbacks) {
17
+ const teardownBrowserLifecycle = subscribeBrowserLifecycle({
18
+ onHidden: callbacks.onHidden,
19
+ onOnline: callbacks.onOnline,
20
+ onPageHide: callbacks.onPageHide,
21
+ onPageShow: callbacks.onPageShow,
22
+ onVisible: callbacks.onVisible
23
+ });
24
+ if (callbacks.onDeviceChange) {
25
+ navigator.mediaDevices?.addEventListener?.(
26
+ "devicechange",
27
+ callbacks.onDeviceChange
28
+ );
29
+ }
30
+ return () => {
31
+ teardownBrowserLifecycle();
32
+ if (callbacks.onDeviceChange) {
33
+ navigator.mediaDevices?.removeEventListener?.(
34
+ "devicechange",
35
+ callbacks.onDeviceChange
36
+ );
37
+ }
38
+ };
39
+ }
40
+ async function replaceMicrophoneTrack(state) {
41
+ if (!state.peerConnection) {
42
+ throw new Error("Realtime transport not active");
43
+ }
44
+ const nextStream = await acquireMicrophoneStream();
45
+ const nextTrack = nextStream.getAudioTracks()[0];
46
+ if (!nextTrack) {
47
+ throw new Error("Microphone access required for Realtime API");
48
+ }
49
+ const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
50
+ if (!sender) {
51
+ throw new Error("No outbound audio sender available");
52
+ }
53
+ await sender.replaceTrack(nextTrack);
54
+ state.mediaStream?.getTracks().forEach((track) => track.stop());
55
+ return {
56
+ audioSender: sender,
57
+ mediaStream: nextStream
58
+ };
59
+ }
60
+ function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
61
+ let pending = null;
62
+ return {
63
+ schedule() {
64
+ if (pending) {
65
+ return;
66
+ }
67
+ pending = setTimeout(() => {
68
+ pending = null;
69
+ onDisconnect();
70
+ }, delayMs);
71
+ },
72
+ cancel() {
73
+ if (!pending) {
74
+ return;
75
+ }
76
+ clearTimeout(pending);
77
+ pending = null;
78
+ }
79
+ };
80
+ }
81
+
82
+ // src/openai-agents/lip-sync-analyzer.ts
83
+ var LipSyncAnalyzer = class {
84
+ constructor(options) {
85
+ this.options = options;
86
+ }
87
+ audioAnalysisStream = null;
88
+ audioContext = null;
89
+ audioSource = null;
90
+ analyser = null;
91
+ lipSyncInterval = null;
92
+ pendingAudioElementPoll = null;
93
+ observedAudioElement = null;
94
+ observedAudioElementListener = null;
95
+ async prepareAudioContext() {
96
+ if (this.audioContext) {
97
+ return;
98
+ }
99
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
100
+ if (!audioContextConstructor) {
101
+ throw new Error("AudioContext is not supported in this browser");
102
+ }
103
+ this.audioContext = new audioContextConstructor();
104
+ }
105
+ observeAudioElement(audioElement) {
106
+ this.detachObservedAudioElement();
107
+ const tryAttachStream = () => {
108
+ const stream = audioElement.srcObject;
109
+ if (stream instanceof MediaStream) {
110
+ this.attachStream(stream);
111
+ }
112
+ };
113
+ this.observedAudioElement = audioElement;
114
+ this.observedAudioElementListener = tryAttachStream;
115
+ audioElement.addEventListener("loadedmetadata", tryAttachStream);
116
+ this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
117
+ }
118
+ attachStream(stream) {
119
+ if (this.audioAnalysisStream === stream) {
120
+ return;
121
+ }
122
+ this.stopAudioElementPolling();
123
+ this.stopLipSyncAnalysis();
124
+ this.audioAnalysisStream = stream;
125
+ try {
126
+ if (!this.audioContext) {
127
+ void this.prepareAudioContext().catch((error) => {
128
+ this.options.onError?.(error);
129
+ });
130
+ }
131
+ if (!this.audioContext) {
132
+ return;
133
+ }
134
+ this.audioSource?.disconnect?.();
135
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
136
+ this.analyser = this.audioContext.createAnalyser();
137
+ this.analyser.fftSize = 256;
138
+ this.analyser.smoothingTimeConstant = 0.8;
139
+ this.audioSource.connect(this.analyser);
140
+ this.startLipSyncAnalysis();
141
+ } catch (error) {
142
+ this.options.onError?.(error);
143
+ }
144
+ }
145
+ stopOutput() {
146
+ this.stopLipSyncAnalysis();
147
+ }
148
+ pause() {
149
+ this.stopLipSyncAnalysis();
150
+ }
151
+ resume() {
152
+ if (!this.analyser || this.lipSyncInterval) {
153
+ return;
154
+ }
155
+ this.startLipSyncAnalysis();
156
+ }
157
+ cleanup() {
158
+ this.stopLipSyncAnalysis();
159
+ this.stopAudioElementPolling();
160
+ this.detachObservedAudioElement();
161
+ this.audioSource?.disconnect?.();
162
+ this.audioSource = null;
163
+ if (this.audioContext) {
164
+ void this.audioContext.close();
165
+ this.audioContext = null;
166
+ }
167
+ this.audioAnalysisStream = null;
168
+ this.analyser = null;
169
+ }
170
+ startLipSyncAnalysis() {
171
+ if (!this.analyser) {
172
+ return;
173
+ }
174
+ const bufferLength = this.analyser.frequencyBinCount;
175
+ const dataArray = new Uint8Array(bufferLength);
176
+ this.lipSyncInterval = window.setInterval(() => {
177
+ if (!this.analyser) {
178
+ return;
179
+ }
180
+ this.analyser.getByteFrequencyData(dataArray);
181
+ let sum = 0;
182
+ for (let index = 0; index < bufferLength; index += 1) {
183
+ const normalized = dataArray[index] / 255;
184
+ sum += normalized * normalized;
185
+ }
186
+ const rms = Math.sqrt(sum / bufferLength);
187
+ this.options.onRms(Math.min(rms * 3, 1));
188
+ }, 1e3 / 60);
189
+ }
190
+ stopLipSyncAnalysis() {
191
+ if (this.lipSyncInterval) {
192
+ clearInterval(this.lipSyncInterval);
193
+ this.lipSyncInterval = null;
194
+ }
195
+ this.options.onRms(0);
196
+ }
197
+ stopAudioElementPolling() {
198
+ if (this.pendingAudioElementPoll) {
199
+ clearInterval(this.pendingAudioElementPoll);
200
+ this.pendingAudioElementPoll = null;
201
+ }
202
+ }
203
+ detachObservedAudioElement() {
204
+ if (this.observedAudioElement && this.observedAudioElementListener) {
205
+ this.observedAudioElement.removeEventListener(
206
+ "loadedmetadata",
207
+ this.observedAudioElementListener
208
+ );
209
+ }
210
+ this.observedAudioElement = null;
211
+ this.observedAudioElementListener = null;
212
+ }
213
+ };
214
+
215
+ // src/openai/defaults.ts
216
+ var DEFAULT_OPENAI_REALTIME_MODEL = "gpt-realtime-mini";
217
+ var DEFAULT_OPENAI_REALTIME_VOICE = "marin";
218
+ var DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
219
+
220
+ export { DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS, DEFAULT_OPENAI_REALTIME_MODEL, DEFAULT_OPENAI_REALTIME_VOICE, LipSyncAnalyzer, acquireMicrophoneStream, bindTransportLifecycle, createIceDisconnectDebouncer, replaceMicrophoneTrack };
221
+ //# sourceMappingURL=chunk-3PIMXNQU.mjs.map
222
+ //# sourceMappingURL=chunk-3PIMXNQU.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/internal/microphone.ts","../src/internal/webrtc-lifecycle.ts","../src/openai-agents/lip-sync-analyzer.ts","../src/openai/defaults.ts"],"names":[],"mappings":";;;;AAAO,IAAM,8BAAA,GAAiC;AAAA,EAC5C,gBAAA,EAAkB,IAAA;AAAA,EAClB,gBAAA,EAAkB,IAAA;AAAA,EAClB,eAAA,EAAiB;AACnB,CAAA;AAEA,eAAsB,uBAAA,GAAgD;AACpE,EAAA,OAAO,SAAA,CAAU,aAAa,YAAA,CAAa;AAAA,IACzC,KAAA,EAAO;AAAA,GACR,CAAA;AACH;ACPO,IAAM,oCAAA,GAAuC;AAW7C,SAAS,uBACd,SAAA,EACY;AACZ,EAAA,MAAM,2BAA2B,yBAAA,CAA0B;AAAA,IACzD,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,UAAU,SAAA,CAAU,QAAA;AAAA,IACpB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,YAAY,SAAA,CAAU,UAAA;AAAA,IACtB,WAAW,SAAA,CAAU;AAAA,GACtB,CAAA;AAED,EAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,IAAA,SAAA,CAAU,YAAA,EAAc,gBAAA;AAAA,MACtB,cAAA;AAAA,MACA,SAAA,CAAU;AAAA,KACZ;AAAA,EACF;AAEA,EAAA,OAAO,MAAM;AACX,IAAA,wBAAA,EAAyB;AACzB,IAAA,IAAI,UAAU,cAAA,EAAgB;AAC5B,MAAA,SAAA,CAAU,YAAA,EAAc,mBAAA;AAAA,QACtB,cAAA;AAAA,QACA,SAAA,CAAU;AAAA,OACZ;AAAA,IACF;AAAA,EACF,CAAA;AACF;AAQA,eAAsB,uBACpB,KAAA,EACkE;AAClE,EAAA,IAAI,CAAC,MAAM,cAAA,EAAgB;AACzB,IAAA,MAAM,IAAI,MAAM,+BAA+B,CAAA;AAAA,EACjD;AAEA,EAAA,MAAM,UAAA,GAAa,MAAM,uBAAA,EAAwB;AACjD,EAAA,MAAM,SAAA,GAAY,UAAA,CAAW,cAAA,EAAe,CAAE,CAAC,CAAA;AAC/C,EAAA,IAAI,CAAC,SAAA,EAAW;AACd,IAAA,MAAM,IAAI,MAAM,6CAA6C,CAAA;AAAA,EAC/D;AAEA,EAAA,MAAM,MAAA,GACJ,MAAM,WAAA,KACL,OAAO,MAAM,cAAA,CAAe,UAAA,KAAe,aACvC,KAAA,CAAM,cAAA,CACJ,YAAW,CACX,IAAA,CAAK,CAAC,SAAA,KAAc,SAAA,CAAU,OAAO,IAAA,KAAS,OAAO,KAAK,IAAA,GAC7D,IAAA,CAAA;AACN,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,EACtD;AAEA,EAAA,MAAM,MAAA,CAAO,aAAa,SAAS,CAAA;AACnC,EAAA,KAAA,CAAM,WAAA,EAAa,WAAU,CAAE,OAAA,CAAQ,CAAC,KAAA,KAAU,KAAA,CAAM,MAAM,CAAA;AAE9D,EAAA,OAAO;AAAA,IACL,WAAA,EAAa,MAAA;AAAA,IACb,WAAA,EAAa;AAAA,GACf;AACF;AAEO,SAAS,4BAAA,CACd,YAAA,EACA,OAAA,GAAU,oCAAA,EAIV;AACA,EAAA,IAAI,OAAA,GAAgD,IAAA;AAEpD,EAAA,OAAO;AAAA,IACL,QAAA,GAAW;AACT,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AAEA,MAAA,OAAA,GAAU,WAAW,MAAM;AACzB,QAAA,OAAA,GAAU,IAAA;AACV,QAAA,YAAA,EAAa;AAAA,MACf,GAAG,OAAO,CAAA;AAAA,IACZ,CAAA;AAAA,IACA,MAAA,GAAS;AACP,MAAA,IAAI,CAAC,OAAA,EAAS;AACZ,QAAA;AAAA,MACF;AAEA,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,OAAA,GAAU,IAAA;AAAA,IACZ;AAAA,GACF;AACF;;;AC1GO,IAAM,kBAAN,MAAsB;AAAA,EAU3B,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAAkC;AAAA,EAT9C,mBAAA,GAA0C,IAAA;AAAA,EAC1C,YAAA,GAAoC,IAAA;AAAA,EACpC,WAAA,GAAiD,IAAA;AAAA,EACjD,QAAA,GAAgC,IAAA;AAAA,EAChC,eAAA,GAAiC,IAAA;AAAA,EACjC,uBAAA,GAAyC,IAAA;AAAA,EACzC,oBAAA,GAAgD,IAAA;AAAA,EAChD,4BAAA,GAAoD,IAAA;AAAA,EAI5D,MAAM,mBAAA,GAAqC;AACzC,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GACJ,MAAA,CAAO,YAAA,IACN,MAAA,CACE,kBAAA;AACL,IAAA,IAAI,CAAC,uBAAA,EAAyB;AAC5B,MAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AAAA,IACjE;AAEA,IAAA,IAAA,CAAK,YAAA,GAAe,IAAI,uBAAA,EAAwB;AAAA,EAClD;AAAA,EAEA,oBAAoB,YAAA,EAAsC;AACxD,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,MAAM,kBAAkB,MAAM;AAC5B,MAAA,MAAM,SAAS,YAAA,CAAa,SAAA;AAC5B,MAAA,IAAI,kBAAkB,WAAA,EAAa;AACjC,QAAA,IAAA,CAAK,aAAa,MAAM,CAAA;AAAA,MAC1B;AAAA,IACF,CAAA;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,YAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,eAAA;AACpC,IAAA,YAAA,CAAa,gBAAA,CAAiB,kBAAkB,eAAe,CAAA;AAC/D,IAAA,IAAA,CAAK,uBAAA,GAA0B,MAAA,CAAO,WAAA,CAAY,eAAA,EAAiB,EAAE,CAAA;AAAA,EACvE;AAAA,EAEA,aAAa,MAAA,EAA2B;AACtC,IAAA,IAAI,IAAA,CAAK,wBAAwB,MAAA,EAAQ;AACvC,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,mBAAA,GAAsB,MAAA;AAE3B,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA,KAAK,IAAA,CAAK,mBAAA,EAAoB,CAAE,KAAA,CAAM,CAAC,KAAA,KAAU;AAC/C,UAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,QAC9B,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,IAAI,CAAC,KAAK,YAAA,EAAc;AACtB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,MAAA,IAAA,CAAK,WAAA,GAAc,IAAA,CAAK,YAAA,CAAa,uBAAA,CAAwB,MAAM,CAAA;AACnE,MAAA,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,YAAA,CAAa,cAAA,EAAe;AACjD,MAAA,IAAA,CAAK,SAAS,OAAA,GAAU,GAAA;AACxB,MAAA,IAAA,CAAK,SAAS,qBAAA,GAAwB,GAAA;AACtC,MAAA,IAAA,CAAK,WAAA,CAAY,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AAEtC,MAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,IAC5B,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,UAAU,KAAK,CAAA;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,UAAA,GAAmB;AACjB,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,KAAA,GAAc;AACZ,IAAA,IAAA,CAAK,mBAAA,EAAoB;AAAA,EAC3B;AAAA,EAEA,MAAA,GAAe;AACb,IAAA,IAAI,CAAC,IAAA,CAAK,QAAA,IAAY,IAAA,CAAK,eAAA,EAAiB;AAC1C,MAAA;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,EAAqB;AAAA,EAC5B;AAAA,EAEA,OAAA,GAAgB;AACd,IAAA,IAAA,CAAK,mBAAA,EAAoB;AACzB,IAAA,IAAA,CAAK,uBAAA,EAAwB;AAC7B,IAAA,IAAA,CAAK,0BAAA,EAA2B;AAEhC,IAAA,IAAA,CAAK,aAAa,UAAA,IAAa;AAC/B,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAEnB,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,KAAK,IAAA,CAAK,aAAa,KAAA,EAAM;AAC7B,MAAA,IAAA,CAAK,YAAA,GAAe,IAAA;AAAA,IACtB;AAEA,IAAA,IAAA,CAAK,mBAAA,GAAsB,IAAA;AAC3B,IAAA,IAAA,CAAK,QAAA,GAAW,IAAA;AAAA,EAClB;AAAA,EAEQ,oBAAA,GAA6B;AACnC,IAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,QAAA,CAAS,iBAAA;AACnC,IAAA,MAAM,SAAA,GAAY,IAAI,UAAA,CAAW,YAAY,CAAA;AAE7C,IAAA,IAAA,CAAK,eAAA,GAAkB,MAAA,CAAO,WAAA,CAAY,MAAM;AAC9C,MAAA,IAAI,CAAC,KAAK,QAAA,EAAU;AAClB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,QAAA,CAAS,qBAAqB,SAAS,CAAA;AAE5C,MAAA,IAAI,GAAA,GAAM,CAAA;AACV,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,YAAA,EAAc,SAAS,CAAA,EAAG;AACpD,QAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAK,CAAA,GAAI,GAAA;AACtC,QAAA,GAAA,IAAO,UAAA,GAAa,UAAA;AAAA,MACtB;AAEA,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,IAAA,CAAK,GAAA,GAAM,YAAY,CAAA;AACxC,MAAA,IAAA,CAAK,QAAQ,KAAA,CAAM,IAAA,CAAK,IAAI,GAAA,GAAM,CAAA,EAAG,CAAC,CAAC,CAAA;AAAA,IACzC,CAAA,EAAG,MAAO,EAAE,CAAA;AAAA,EACd;AAAA,EAEQ,mBAAA,GAA4B;AAClC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,aAAA,CAAc,KAAK,eAAe,CAAA;AAClC,MAAA,IAAA,CAAK,eAAA,GAAkB,IAAA;AAAA,IACzB;AACA,IAAA,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAC,CAAA;AAAA,EACtB;AAAA,EAEQ,uBAAA,GAAgC;AACtC,IAAA,IAAI,KAAK,uBAAA,EAAyB;AAChC,MAAA,aAAA,CAAc,KAAK,uBAAuB,CAAA;AAC1C,MAAA,IAAA,CAAK,uBAAA,GAA0B,IAAA;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,0BAAA,GAAmC;AACzC,IAAA,IAAI,IAAA,CAAK,oBAAA,IAAwB,IAAA,CAAK,4BAAA,EAA8B;AAClE,MAAA,IAAA,CAAK,oBAAA,CAAqB,mBAAA;AAAA,QACxB,gBAAA;AAAA,QACA,IAAA,CAAK;AAAA,OACP;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,oBAAA,GAAuB,IAAA;AAC5B,IAAA,IAAA,CAAK,4BAAA,GAA+B,IAAA;AAAA,EACtC;AACF;;;ACnKO,IAAM,6BAAA,GAAgC;AACtC,IAAM,6BAAA,GAAgC;AACtC,IAAM,0CAAA,GACX","file":"chunk-3PIMXNQU.mjs","sourcesContent":["export const DEFAULT_MICROPHONE_CONSTRAINTS = {\n echoCancellation: true,\n noiseSuppression: true,\n autoGainControl: true,\n} satisfies MediaTrackConstraints;\n\nexport async function acquireMicrophoneStream(): Promise<MediaStream> {\n return navigator.mediaDevices.getUserMedia({\n audio: DEFAULT_MICROPHONE_CONSTRAINTS,\n });\n}\n","import { subscribeBrowserLifecycle } from \"@charivo/core\";\nimport { acquireMicrophoneStream } from \"./microphone\";\n\nexport const DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1_000;\n\nexport interface TransportLifecycleCallbacks {\n onHidden?(): void;\n onOnline?(): void;\n onPageHide?(): void;\n onPageShow?(event: PageTransitionEvent): void;\n onVisible?(): void;\n onDeviceChange?(): void;\n}\n\nexport function bindTransportLifecycle(\n callbacks: TransportLifecycleCallbacks,\n): () => void {\n const teardownBrowserLifecycle = subscribeBrowserLifecycle({\n onHidden: callbacks.onHidden,\n onOnline: callbacks.onOnline,\n onPageHide: callbacks.onPageHide,\n onPageShow: callbacks.onPageShow,\n onVisible: callbacks.onVisible,\n });\n\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.addEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n\n return () => {\n teardownBrowserLifecycle();\n if (callbacks.onDeviceChange) {\n navigator.mediaDevices?.removeEventListener?.(\n \"devicechange\",\n callbacks.onDeviceChange,\n );\n }\n };\n}\n\nexport interface AudioSenderTrackState {\n audioSender: RTCRtpSender | null;\n mediaStream: MediaStream | null;\n peerConnection: RTCPeerConnection | null;\n}\n\nexport async function replaceMicrophoneTrack(\n state: AudioSenderTrackState,\n): Promise<{ audioSender: RTCRtpSender; mediaStream: MediaStream }> {\n if (!state.peerConnection) {\n throw new Error(\"Realtime transport not active\");\n }\n\n const nextStream = await acquireMicrophoneStream();\n const nextTrack = nextStream.getAudioTracks()[0];\n if (!nextTrack) {\n throw new Error(\"Microphone access required for Realtime API\");\n }\n\n const sender =\n state.audioSender ??\n (typeof state.peerConnection.getSenders === \"function\"\n ? (state.peerConnection\n .getSenders()\n .find((candidate) => candidate.track?.kind === \"audio\") ?? null)\n : null);\n if (!sender) {\n throw new Error(\"No outbound audio sender available\");\n }\n\n await sender.replaceTrack(nextTrack);\n state.mediaStream?.getTracks().forEach((track) => track.stop());\n\n return {\n audioSender: sender,\n mediaStream: nextStream,\n };\n}\n\nexport function createIceDisconnectDebouncer(\n onDisconnect: () => void,\n delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS,\n): {\n schedule(): void;\n cancel(): void;\n} {\n let pending: ReturnType<typeof setTimeout> | null = null;\n\n return {\n schedule() {\n if (pending) {\n return;\n }\n\n pending = setTimeout(() => {\n pending = null;\n onDisconnect();\n }, delayMs);\n },\n cancel() {\n if (!pending) {\n return;\n }\n\n clearTimeout(pending);\n pending = null;\n },\n };\n}\n","interface LipSyncAnalyzerOptions {\n onRms: (rms: number) => void;\n onError?: (error: unknown) => void;\n}\n\nexport class LipSyncAnalyzer {\n private audioAnalysisStream: MediaStream | null = null;\n private audioContext: AudioContext | null = null;\n private audioSource: MediaStreamAudioSourceNode | null = null;\n private analyser: AnalyserNode | null = null;\n private lipSyncInterval: number | null = null;\n private pendingAudioElementPoll: number | null = null;\n private observedAudioElement: HTMLAudioElement | null = null;\n private observedAudioElementListener: (() => void) | null = null;\n\n constructor(private options: LipSyncAnalyzerOptions) {}\n\n async prepareAudioContext(): Promise<void> {\n if (this.audioContext) {\n return;\n }\n\n const audioContextConstructor =\n window.AudioContext ||\n (window as Window & { webkitAudioContext?: typeof AudioContext })\n .webkitAudioContext;\n if (!audioContextConstructor) {\n throw new Error(\"AudioContext is not supported in this browser\");\n }\n\n this.audioContext = new audioContextConstructor();\n }\n\n observeAudioElement(audioElement: HTMLAudioElement): void {\n this.detachObservedAudioElement();\n\n const tryAttachStream = () => {\n const stream = audioElement.srcObject;\n if (stream instanceof MediaStream) {\n this.attachStream(stream);\n }\n };\n\n this.observedAudioElement = audioElement;\n this.observedAudioElementListener = tryAttachStream;\n audioElement.addEventListener(\"loadedmetadata\", tryAttachStream);\n this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);\n }\n\n attachStream(stream: MediaStream): void {\n if (this.audioAnalysisStream === stream) {\n return;\n }\n\n this.stopAudioElementPolling();\n this.stopLipSyncAnalysis();\n this.audioAnalysisStream = stream;\n\n try {\n if (!this.audioContext) {\n void this.prepareAudioContext().catch((error) => {\n this.options.onError?.(error);\n });\n }\n\n if (!this.audioContext) {\n return;\n }\n\n this.audioSource?.disconnect?.();\n this.audioSource = this.audioContext.createMediaStreamSource(stream);\n this.analyser = this.audioContext.createAnalyser();\n this.analyser.fftSize = 256;\n this.analyser.smoothingTimeConstant = 0.8;\n this.audioSource.connect(this.analyser);\n\n this.startLipSyncAnalysis();\n } catch (error) {\n this.options.onError?.(error);\n }\n }\n\n stopOutput(): void {\n this.stopLipSyncAnalysis();\n }\n\n pause(): void {\n this.stopLipSyncAnalysis();\n }\n\n resume(): void {\n if (!this.analyser || this.lipSyncInterval) {\n return;\n }\n\n this.startLipSyncAnalysis();\n }\n\n cleanup(): void {\n this.stopLipSyncAnalysis();\n this.stopAudioElementPolling();\n this.detachObservedAudioElement();\n\n this.audioSource?.disconnect?.();\n this.audioSource = null;\n\n if (this.audioContext) {\n void this.audioContext.close();\n this.audioContext = null;\n }\n\n this.audioAnalysisStream = null;\n this.analyser = null;\n }\n\n private startLipSyncAnalysis(): void {\n if (!this.analyser) {\n return;\n }\n\n const bufferLength = this.analyser.frequencyBinCount;\n const dataArray = new Uint8Array(bufferLength);\n\n this.lipSyncInterval = window.setInterval(() => {\n if (!this.analyser) {\n return;\n }\n\n this.analyser.getByteFrequencyData(dataArray);\n\n let sum = 0;\n for (let index = 0; index < bufferLength; index += 1) {\n const normalized = dataArray[index] / 255;\n sum += normalized * normalized;\n }\n\n const rms = Math.sqrt(sum / bufferLength);\n this.options.onRms(Math.min(rms * 3, 1));\n }, 1000 / 60);\n }\n\n private stopLipSyncAnalysis(): void {\n if (this.lipSyncInterval) {\n clearInterval(this.lipSyncInterval);\n this.lipSyncInterval = null;\n }\n this.options.onRms(0);\n }\n\n private stopAudioElementPolling(): void {\n if (this.pendingAudioElementPoll) {\n clearInterval(this.pendingAudioElementPoll);\n this.pendingAudioElementPoll = null;\n }\n }\n\n private detachObservedAudioElement(): void {\n if (this.observedAudioElement && this.observedAudioElementListener) {\n this.observedAudioElement.removeEventListener(\n \"loadedmetadata\",\n this.observedAudioElementListener,\n );\n }\n\n this.observedAudioElement = null;\n this.observedAudioElementListener = null;\n }\n}\n","import { DEFAULT_REALTIME_AGENT_INSTRUCTIONS as DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS } from \"../instructions\";\n\n// Mirrors packages/server/src/openai/realtime/index.ts until the server/browser\n// OpenAI defaults consolidate under ROADMAP P0-ARCH-2.\nexport const DEFAULT_OPENAI_REALTIME_MODEL = \"gpt-realtime-mini\";\nexport const DEFAULT_OPENAI_REALTIME_VOICE = \"marin\";\nexport const DEFAULT_OPENAI_REALTIME_AGENT_INSTRUCTIONS =\n DEFAULT_GENERIC_REALTIME_AGENT_INSTRUCTIONS;\n"]}
@@ -1,5 +1,6 @@
1
- import { createIceDisconnectDebouncer, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, acquireMicrophoneStream, bindTransportLifecycle, replaceMicrophoneTrack, DEFAULT_OPENAI_REALTIME_VOICE } from './chunk-E656Y2PO.mjs';
2
- import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-X6YRHDTW.mjs';
1
+ import { delay } from './chunk-XPKFJWHL.mjs';
2
+ import { LipSyncAnalyzer, createIceDisconnectDebouncer, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS, acquireMicrophoneStream, bindTransportLifecycle, replaceMicrophoneTrack, DEFAULT_OPENAI_REALTIME_VOICE } from './chunk-3PIMXNQU.mjs';
3
+ import { isRecord, fetchWithTimeout, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap } from './chunk-PNBYFAER.mjs';
3
4
  import { OPENAI_REALTIME_ADAPTER } from '@charivo/core';
4
5
 
5
6
  var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
@@ -21,10 +22,10 @@ var OpenAIRealtimeClient = class {
21
22
  audioElement = null;
22
23
  mediaStream = null;
23
24
  audioSender = null;
24
- audioContext = null;
25
- audioSource = null;
26
- analyser = null;
27
- lipSyncInterval = null;
25
+ lipSyncAnalyzer = new LipSyncAnalyzer({
26
+ onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
27
+ onError: (error) => console.error("Failed to setup audio analysis:", error)
28
+ });
28
29
  isResponseInProgress = false;
29
30
  cancelInFlight = false;
30
31
  hasStartedAssistantResponse = false;
@@ -56,14 +57,14 @@ var OpenAIRealtimeClient = class {
56
57
  this.audioElement = document.createElement("audio");
57
58
  this.audioElement.autoplay = true;
58
59
  this.audioElement.setAttribute("playsinline", "true");
59
- this.prepareAudioAnalysis();
60
+ void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
60
61
  this.bindConnectionEvents();
61
62
  this.bindTransportLifecycleEvents();
62
63
  this.pc.ontrack = (event) => {
63
64
  if (this.audioElement) {
64
65
  this.audioElement.srcObject = event.streams[0];
65
66
  }
66
- this.setupAudioAnalysis(event.streams[0]);
67
+ this.lipSyncAnalyzer.attachStream(event.streams[0]);
67
68
  };
68
69
  try {
69
70
  this.mediaStream = await acquireMicrophoneStream();
@@ -90,14 +91,10 @@ var OpenAIRealtimeClient = class {
90
91
  session: config ?? {},
91
92
  sdpOffer: offer.sdp
92
93
  });
93
- if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
94
- throw new Error(
95
- `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
96
- );
97
- }
94
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
98
95
  await this.pc.setRemoteDescription({
99
96
  type: "answer",
100
- sdp: bootstrap.answerSdp
97
+ sdp: answerSdp
101
98
  });
102
99
  this.emitEvent({ type: "session.started" });
103
100
  } catch (error) {
@@ -286,10 +283,7 @@ var OpenAIRealtimeClient = class {
286
283
  if (this.cancelInFlight || !event.delta) {
287
284
  return;
288
285
  }
289
- if (!this.hasStartedAssistantResponse) {
290
- this.hasStartedAssistantResponse = true;
291
- this.emitEvent({ type: "assistant.response.started" });
292
- }
286
+ this.ensureAssistantResponseStarted();
293
287
  this.assistantText += event.delta;
294
288
  this.emitEvent({
295
289
  type: "assistant.text.delta",
@@ -300,39 +294,13 @@ var OpenAIRealtimeClient = class {
300
294
  if (this.cancelInFlight || !event.text) {
301
295
  return;
302
296
  }
303
- if (!this.hasStartedAssistantResponse) {
304
- this.hasStartedAssistantResponse = true;
305
- this.emitEvent({ type: "assistant.response.started" });
306
- }
307
- if (event.text !== this.assistantText) {
308
- const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
309
- this.assistantText = event.text;
310
- if (delta) {
311
- this.emitEvent({
312
- type: "assistant.text.delta",
313
- text: delta
314
- });
315
- }
316
- }
297
+ this.emitFinalAssistantText(event.text);
317
298
  return;
318
299
  case "response.output_audio_transcript.done":
319
300
  if (this.cancelInFlight || !event.transcript) {
320
301
  return;
321
302
  }
322
- if (!this.hasStartedAssistantResponse) {
323
- this.hasStartedAssistantResponse = true;
324
- this.emitEvent({ type: "assistant.response.started" });
325
- }
326
- if (event.transcript !== this.assistantText) {
327
- const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
328
- this.assistantText = event.transcript;
329
- if (delta) {
330
- this.emitEvent({
331
- type: "assistant.text.delta",
332
- text: delta
333
- });
334
- }
335
- }
303
+ this.emitFinalAssistantText(event.transcript);
336
304
  return;
337
305
  case "conversation.item.input_audio_transcription.completed":
338
306
  if (event.transcript) {
@@ -463,17 +431,17 @@ var OpenAIRealtimeClient = class {
463
431
  this.emitConnectionLost("online");
464
432
  };
465
433
  handleHidden = () => {
466
- this.pauseAudioAnalysis();
434
+ this.lipSyncAnalyzer.pause();
467
435
  };
468
436
  handleVisible = () => {
469
- this.resumeAudioAnalysis();
437
+ this.lipSyncAnalyzer.resume();
470
438
  this.emitConnectionLost("visibility");
471
439
  };
472
440
  handlePageHide = () => {
473
- this.pauseAudioAnalysis();
441
+ this.lipSyncAnalyzer.pause();
474
442
  };
475
443
  handlePageShow = (event) => {
476
- this.resumeAudioAnalysis();
444
+ this.lipSyncAnalyzer.resume();
477
445
  if (event.persisted) {
478
446
  this.emitConnectionLost("pageshow");
479
447
  }
@@ -508,7 +476,7 @@ var OpenAIRealtimeClient = class {
508
476
  this.isResponseInProgress = false;
509
477
  this.cancelInFlight = false;
510
478
  this.resetResponseTracking();
511
- this.stopLipSyncAnalysis();
479
+ this.lipSyncAnalyzer.stopOutput();
512
480
  this.emitEvent({
513
481
  type: "connection.lost",
514
482
  cause,
@@ -524,77 +492,11 @@ var OpenAIRealtimeClient = class {
524
492
  }
525
493
  return true;
526
494
  }
527
- prepareAudioAnalysis() {
528
- if (this.audioContext) {
529
- return;
530
- }
531
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
532
- if (!audioContextConstructor) {
533
- return;
534
- }
535
- this.audioContext = new audioContextConstructor();
536
- }
537
- setupAudioAnalysis(stream) {
538
- try {
539
- this.prepareAudioAnalysis();
540
- if (!this.audioContext) {
541
- return;
542
- }
543
- this.audioSource?.disconnect();
544
- this.audioSource = this.audioContext.createMediaStreamSource(stream);
545
- this.analyser = this.audioContext.createAnalyser();
546
- this.analyser.fftSize = 256;
547
- this.analyser.smoothingTimeConstant = 0.8;
548
- this.audioSource.connect(this.analyser);
549
- this.startLipSyncAnalysis();
550
- } catch (error) {
551
- console.error("Failed to setup audio analysis:", error);
552
- }
553
- }
554
- startLipSyncAnalysis() {
555
- if (!this.analyser) {
556
- return;
557
- }
558
- const bufferLength = this.analyser.frequencyBinCount;
559
- const dataArray = new Uint8Array(bufferLength);
560
- this.lipSyncInterval = window.setInterval(() => {
561
- if (!this.analyser) {
562
- return;
563
- }
564
- this.analyser.getByteFrequencyData(dataArray);
565
- let sum = 0;
566
- for (let index = 0; index < bufferLength; index += 1) {
567
- const normalized = dataArray[index] / 255;
568
- sum += normalized * normalized;
569
- }
570
- const rms = Math.sqrt(sum / bufferLength);
571
- this.emitEvent({
572
- type: "audio.lipsync",
573
- rms: Math.min(rms * 3, 1)
574
- });
575
- }, 1e3 / 60);
576
- }
577
- stopLipSyncAnalysis() {
578
- if (this.lipSyncInterval) {
579
- clearInterval(this.lipSyncInterval);
580
- this.lipSyncInterval = null;
581
- }
582
- this.emitEvent({ type: "audio.lipsync", rms: 0 });
583
- }
584
- pauseAudioAnalysis() {
585
- this.stopLipSyncAnalysis();
586
- }
587
- resumeAudioAnalysis() {
588
- if (!this.analyser || this.lipSyncInterval) {
589
- return;
590
- }
591
- this.startLipSyncAnalysis();
592
- }
593
495
  cleanup() {
594
496
  this.isCleaningUp = true;
595
497
  this.unbindTransportLifecycleEvents();
596
498
  this.iceDisconnectDebouncer.cancel();
597
- this.stopLipSyncAnalysis();
499
+ this.lipSyncAnalyzer.cleanup();
598
500
  this.rejectPendingSessionUpdate(
599
501
  new Error("Realtime session ended before session update completed")
600
502
  );
@@ -614,13 +516,7 @@ var OpenAIRealtimeClient = class {
614
516
  this.audioElement.srcObject = null;
615
517
  this.audioElement = null;
616
518
  }
617
- if (this.audioContext) {
618
- this.audioContext.close();
619
- this.audioContext = null;
620
- }
621
- this.audioSource = null;
622
519
  this.audioSender = null;
623
- this.analyser = null;
624
520
  this.isResponseInProgress = false;
625
521
  this.cancelInFlight = false;
626
522
  this.connectionLossNotified = false;
@@ -668,14 +564,10 @@ var OpenAIRealtimeClient = class {
668
564
  session: config ?? this.currentSessionConfig ?? {},
669
565
  sdpOffer: offer.sdp
670
566
  });
671
- if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
672
- throw new Error(
673
- `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
674
- );
675
- }
567
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
676
568
  await pc.setRemoteDescription({
677
569
  type: "answer",
678
- sdp: bootstrap.answerSdp
570
+ sdp: answerSdp
679
571
  });
680
572
  return true;
681
573
  }
@@ -713,11 +605,38 @@ var OpenAIRealtimeClient = class {
713
605
  getSessionUpdateTimeoutMs() {
714
606
  return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
715
607
  }
608
+ resolveWebRTCAnswerSdp(bootstrap) {
609
+ if (bootstrap.adapter !== OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
610
+ throw new Error(
611
+ `OpenAI realtime client only supports ${OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
612
+ );
613
+ }
614
+ return bootstrap.answerSdp;
615
+ }
716
616
  resetResponseTracking() {
717
617
  this.assistantText = "";
718
618
  this.hasStartedAssistantResponse = false;
719
619
  this.hasStartedAudioOutput = false;
720
620
  }
621
+ ensureAssistantResponseStarted() {
622
+ if (!this.hasStartedAssistantResponse) {
623
+ this.hasStartedAssistantResponse = true;
624
+ this.emitEvent({ type: "assistant.response.started" });
625
+ }
626
+ }
627
+ emitFinalAssistantText(full) {
628
+ this.ensureAssistantResponseStarted();
629
+ if (full !== this.assistantText) {
630
+ const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
631
+ this.assistantText = full;
632
+ if (delta) {
633
+ this.emitEvent({
634
+ type: "assistant.text.delta",
635
+ text: delta
636
+ });
637
+ }
638
+ }
639
+ }
721
640
  emitEvent(event) {
722
641
  for (const callback of this.eventCallbacks) {
723
642
  callback(event);
@@ -739,12 +658,21 @@ function createOpenAIRealtimeClient(options) {
739
658
  return new OpenAIRealtimeClient(options);
740
659
  }
741
660
  function toOpenAIRealtimeSessionUpdate(config) {
661
+ const audio = {
662
+ output: {
663
+ voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
664
+ }
665
+ };
666
+ const transcription = config?.inputAudioTranscription;
667
+ if (transcription !== void 0) {
668
+ if (transcription.enabled === false) {
669
+ audio.input = { transcription: null };
670
+ } else if (transcription.model !== void 0) {
671
+ audio.input = { transcription: { model: transcription.model } };
672
+ }
673
+ }
742
674
  const session = {
743
- audio: {
744
- output: {
745
- voice: config?.voice ?? DEFAULT_OPENAI_REALTIME_VOICE
746
- }
747
- },
675
+ audio,
748
676
  tool_choice: config?.toolChoice ?? "auto"
749
677
  };
750
678
  if (config?.instructions) {
@@ -756,17 +684,12 @@ function toOpenAIRealtimeSessionUpdate(config) {
756
684
  if (config?.maxTokens !== void 0) {
757
685
  session.max_response_output_tokens = config.maxTokens;
758
686
  }
759
- if (config?.tools?.length) {
760
- session.tools = config.tools;
687
+ if (config && "tools" in config) {
688
+ session.tools = config.tools ?? [];
761
689
  }
762
690
  return session;
763
691
  }
764
- function delay(ms) {
765
- return new Promise((resolve) => {
766
- setTimeout(resolve, ms);
767
- });
768
- }
769
692
 
770
693
  export { OpenAIRealtimeClient, createOpenAIRealtimeClient };
771
- //# sourceMappingURL=chunk-FRTYFCPV.mjs.map
772
- //# sourceMappingURL=chunk-FRTYFCPV.mjs.map
694
+ //# sourceMappingURL=chunk-MI4QTRWT.mjs.map
695
+ //# sourceMappingURL=chunk-MI4QTRWT.mjs.map