@charivo/realtime 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -120,6 +120,146 @@ function isAbortError(error) {
120
120
  return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
121
121
  }
122
122
 
123
+ // src/internal/timing.ts
124
+ function delay(ms) {
125
+ return new Promise((resolve) => {
126
+ setTimeout(resolve, ms);
127
+ });
128
+ }
129
+
130
+ // src/openai-agents/lip-sync-analyzer.ts
131
+ var LipSyncAnalyzer = class {
132
+ constructor(options) {
133
+ this.options = options;
134
+ }
135
+ audioAnalysisStream = null;
136
+ audioContext = null;
137
+ audioSource = null;
138
+ analyser = null;
139
+ lipSyncInterval = null;
140
+ pendingAudioElementPoll = null;
141
+ observedAudioElement = null;
142
+ observedAudioElementListener = null;
143
+ async prepareAudioContext() {
144
+ if (this.audioContext) {
145
+ return;
146
+ }
147
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
148
+ if (!audioContextConstructor) {
149
+ throw new Error("AudioContext is not supported in this browser");
150
+ }
151
+ this.audioContext = new audioContextConstructor();
152
+ }
153
+ observeAudioElement(audioElement) {
154
+ this.detachObservedAudioElement();
155
+ const tryAttachStream = () => {
156
+ const stream = audioElement.srcObject;
157
+ if (stream instanceof MediaStream) {
158
+ this.attachStream(stream);
159
+ }
160
+ };
161
+ this.observedAudioElement = audioElement;
162
+ this.observedAudioElementListener = tryAttachStream;
163
+ audioElement.addEventListener("loadedmetadata", tryAttachStream);
164
+ this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
165
+ }
166
+ attachStream(stream) {
167
+ if (this.audioAnalysisStream === stream) {
168
+ return;
169
+ }
170
+ this.stopAudioElementPolling();
171
+ this.stopLipSyncAnalysis();
172
+ this.audioAnalysisStream = stream;
173
+ try {
174
+ if (!this.audioContext) {
175
+ void this.prepareAudioContext().catch((error) => {
176
+ this.options.onError?.(error);
177
+ });
178
+ }
179
+ if (!this.audioContext) {
180
+ return;
181
+ }
182
+ this.audioSource?.disconnect?.();
183
+ this.audioSource = this.audioContext.createMediaStreamSource(stream);
184
+ this.analyser = this.audioContext.createAnalyser();
185
+ this.analyser.fftSize = 256;
186
+ this.analyser.smoothingTimeConstant = 0.8;
187
+ this.audioSource.connect(this.analyser);
188
+ this.startLipSyncAnalysis();
189
+ } catch (error) {
190
+ this.options.onError?.(error);
191
+ }
192
+ }
193
+ stopOutput() {
194
+ this.stopLipSyncAnalysis();
195
+ }
196
+ pause() {
197
+ this.stopLipSyncAnalysis();
198
+ }
199
+ resume() {
200
+ if (!this.analyser || this.lipSyncInterval) {
201
+ return;
202
+ }
203
+ this.startLipSyncAnalysis();
204
+ }
205
+ cleanup() {
206
+ this.stopLipSyncAnalysis();
207
+ this.stopAudioElementPolling();
208
+ this.detachObservedAudioElement();
209
+ this.audioSource?.disconnect?.();
210
+ this.audioSource = null;
211
+ if (this.audioContext) {
212
+ void this.audioContext.close();
213
+ this.audioContext = null;
214
+ }
215
+ this.audioAnalysisStream = null;
216
+ this.analyser = null;
217
+ }
218
+ startLipSyncAnalysis() {
219
+ if (!this.analyser) {
220
+ return;
221
+ }
222
+ const bufferLength = this.analyser.frequencyBinCount;
223
+ const dataArray = new Uint8Array(bufferLength);
224
+ this.lipSyncInterval = window.setInterval(() => {
225
+ if (!this.analyser) {
226
+ return;
227
+ }
228
+ this.analyser.getByteFrequencyData(dataArray);
229
+ let sum = 0;
230
+ for (let index = 0; index < bufferLength; index += 1) {
231
+ const normalized = dataArray[index] / 255;
232
+ sum += normalized * normalized;
233
+ }
234
+ const rms = Math.sqrt(sum / bufferLength);
235
+ this.options.onRms(Math.min(rms * 3, 1));
236
+ }, 1e3 / 60);
237
+ }
238
+ stopLipSyncAnalysis() {
239
+ if (this.lipSyncInterval) {
240
+ clearInterval(this.lipSyncInterval);
241
+ this.lipSyncInterval = null;
242
+ }
243
+ this.options.onRms(0);
244
+ }
245
+ stopAudioElementPolling() {
246
+ if (this.pendingAudioElementPoll) {
247
+ clearInterval(this.pendingAudioElementPoll);
248
+ this.pendingAudioElementPoll = null;
249
+ }
250
+ }
251
+ detachObservedAudioElement() {
252
+ if (this.observedAudioElement && this.observedAudioElementListener) {
253
+ this.observedAudioElement.removeEventListener(
254
+ "loadedmetadata",
255
+ this.observedAudioElementListener
256
+ );
257
+ }
258
+ this.observedAudioElement = null;
259
+ this.observedAudioElementListener = null;
260
+ }
261
+ };
262
+
123
263
  // src/openai/defaults.ts
124
264
  var DEFAULT_OPENAI_REALTIME_VOICE = "marin";
125
265
 
@@ -143,10 +283,10 @@ var OpenAIRealtimeClient = class {
143
283
  audioElement = null;
144
284
  mediaStream = null;
145
285
  audioSender = null;
146
- audioContext = null;
147
- audioSource = null;
148
- analyser = null;
149
- lipSyncInterval = null;
286
+ lipSyncAnalyzer = new LipSyncAnalyzer({
287
+ onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
288
+ onError: (error) => console.error("Failed to setup audio analysis:", error)
289
+ });
150
290
  isResponseInProgress = false;
151
291
  cancelInFlight = false;
152
292
  hasStartedAssistantResponse = false;
@@ -178,14 +318,14 @@ var OpenAIRealtimeClient = class {
178
318
  this.audioElement = document.createElement("audio");
179
319
  this.audioElement.autoplay = true;
180
320
  this.audioElement.setAttribute("playsinline", "true");
181
- this.prepareAudioAnalysis();
321
+ void this.lipSyncAnalyzer.prepareAudioContext().catch(() => void 0);
182
322
  this.bindConnectionEvents();
183
323
  this.bindTransportLifecycleEvents();
184
324
  this.pc.ontrack = (event) => {
185
325
  if (this.audioElement) {
186
326
  this.audioElement.srcObject = event.streams[0];
187
327
  }
188
- this.setupAudioAnalysis(event.streams[0]);
328
+ this.lipSyncAnalyzer.attachStream(event.streams[0]);
189
329
  };
190
330
  try {
191
331
  this.mediaStream = await acquireMicrophoneStream();
@@ -212,14 +352,10 @@ var OpenAIRealtimeClient = class {
212
352
  session: config ?? {},
213
353
  sdpOffer: offer.sdp
214
354
  });
215
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
216
- throw new Error(
217
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
218
- );
219
- }
355
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
220
356
  await this.pc.setRemoteDescription({
221
357
  type: "answer",
222
- sdp: bootstrap.answerSdp
358
+ sdp: answerSdp
223
359
  });
224
360
  this.emitEvent({ type: "session.started" });
225
361
  } catch (error) {
@@ -408,10 +544,7 @@ var OpenAIRealtimeClient = class {
408
544
  if (this.cancelInFlight || !event.delta) {
409
545
  return;
410
546
  }
411
- if (!this.hasStartedAssistantResponse) {
412
- this.hasStartedAssistantResponse = true;
413
- this.emitEvent({ type: "assistant.response.started" });
414
- }
547
+ this.ensureAssistantResponseStarted();
415
548
  this.assistantText += event.delta;
416
549
  this.emitEvent({
417
550
  type: "assistant.text.delta",
@@ -422,39 +555,13 @@ var OpenAIRealtimeClient = class {
422
555
  if (this.cancelInFlight || !event.text) {
423
556
  return;
424
557
  }
425
- if (!this.hasStartedAssistantResponse) {
426
- this.hasStartedAssistantResponse = true;
427
- this.emitEvent({ type: "assistant.response.started" });
428
- }
429
- if (event.text !== this.assistantText) {
430
- const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
431
- this.assistantText = event.text;
432
- if (delta) {
433
- this.emitEvent({
434
- type: "assistant.text.delta",
435
- text: delta
436
- });
437
- }
438
- }
558
+ this.emitFinalAssistantText(event.text);
439
559
  return;
440
560
  case "response.output_audio_transcript.done":
441
561
  if (this.cancelInFlight || !event.transcript) {
442
562
  return;
443
563
  }
444
- if (!this.hasStartedAssistantResponse) {
445
- this.hasStartedAssistantResponse = true;
446
- this.emitEvent({ type: "assistant.response.started" });
447
- }
448
- if (event.transcript !== this.assistantText) {
449
- const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
450
- this.assistantText = event.transcript;
451
- if (delta) {
452
- this.emitEvent({
453
- type: "assistant.text.delta",
454
- text: delta
455
- });
456
- }
457
- }
564
+ this.emitFinalAssistantText(event.transcript);
458
565
  return;
459
566
  case "conversation.item.input_audio_transcription.completed":
460
567
  if (event.transcript) {
@@ -585,17 +692,17 @@ var OpenAIRealtimeClient = class {
585
692
  this.emitConnectionLost("online");
586
693
  };
587
694
  handleHidden = () => {
588
- this.pauseAudioAnalysis();
695
+ this.lipSyncAnalyzer.pause();
589
696
  };
590
697
  handleVisible = () => {
591
- this.resumeAudioAnalysis();
698
+ this.lipSyncAnalyzer.resume();
592
699
  this.emitConnectionLost("visibility");
593
700
  };
594
701
  handlePageHide = () => {
595
- this.pauseAudioAnalysis();
702
+ this.lipSyncAnalyzer.pause();
596
703
  };
597
704
  handlePageShow = (event) => {
598
- this.resumeAudioAnalysis();
705
+ this.lipSyncAnalyzer.resume();
599
706
  if (event.persisted) {
600
707
  this.emitConnectionLost("pageshow");
601
708
  }
@@ -630,7 +737,7 @@ var OpenAIRealtimeClient = class {
630
737
  this.isResponseInProgress = false;
631
738
  this.cancelInFlight = false;
632
739
  this.resetResponseTracking();
633
- this.stopLipSyncAnalysis();
740
+ this.lipSyncAnalyzer.stopOutput();
634
741
  this.emitEvent({
635
742
  type: "connection.lost",
636
743
  cause,
@@ -646,77 +753,11 @@ var OpenAIRealtimeClient = class {
646
753
  }
647
754
  return true;
648
755
  }
649
- prepareAudioAnalysis() {
650
- if (this.audioContext) {
651
- return;
652
- }
653
- const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
654
- if (!audioContextConstructor) {
655
- return;
656
- }
657
- this.audioContext = new audioContextConstructor();
658
- }
659
- setupAudioAnalysis(stream) {
660
- try {
661
- this.prepareAudioAnalysis();
662
- if (!this.audioContext) {
663
- return;
664
- }
665
- this.audioSource?.disconnect();
666
- this.audioSource = this.audioContext.createMediaStreamSource(stream);
667
- this.analyser = this.audioContext.createAnalyser();
668
- this.analyser.fftSize = 256;
669
- this.analyser.smoothingTimeConstant = 0.8;
670
- this.audioSource.connect(this.analyser);
671
- this.startLipSyncAnalysis();
672
- } catch (error) {
673
- console.error("Failed to setup audio analysis:", error);
674
- }
675
- }
676
- startLipSyncAnalysis() {
677
- if (!this.analyser) {
678
- return;
679
- }
680
- const bufferLength = this.analyser.frequencyBinCount;
681
- const dataArray = new Uint8Array(bufferLength);
682
- this.lipSyncInterval = window.setInterval(() => {
683
- if (!this.analyser) {
684
- return;
685
- }
686
- this.analyser.getByteFrequencyData(dataArray);
687
- let sum = 0;
688
- for (let index = 0; index < bufferLength; index += 1) {
689
- const normalized = dataArray[index] / 255;
690
- sum += normalized * normalized;
691
- }
692
- const rms = Math.sqrt(sum / bufferLength);
693
- this.emitEvent({
694
- type: "audio.lipsync",
695
- rms: Math.min(rms * 3, 1)
696
- });
697
- }, 1e3 / 60);
698
- }
699
- stopLipSyncAnalysis() {
700
- if (this.lipSyncInterval) {
701
- clearInterval(this.lipSyncInterval);
702
- this.lipSyncInterval = null;
703
- }
704
- this.emitEvent({ type: "audio.lipsync", rms: 0 });
705
- }
706
- pauseAudioAnalysis() {
707
- this.stopLipSyncAnalysis();
708
- }
709
- resumeAudioAnalysis() {
710
- if (!this.analyser || this.lipSyncInterval) {
711
- return;
712
- }
713
- this.startLipSyncAnalysis();
714
- }
715
756
  cleanup() {
716
757
  this.isCleaningUp = true;
717
758
  this.unbindTransportLifecycleEvents();
718
759
  this.iceDisconnectDebouncer.cancel();
719
- this.stopLipSyncAnalysis();
760
+ this.lipSyncAnalyzer.cleanup();
720
761
  this.rejectPendingSessionUpdate(
721
762
  new Error("Realtime session ended before session update completed")
722
763
  );
@@ -736,13 +777,7 @@ var OpenAIRealtimeClient = class {
736
777
  this.audioElement.srcObject = null;
737
778
  this.audioElement = null;
738
779
  }
739
- if (this.audioContext) {
740
- this.audioContext.close();
741
- this.audioContext = null;
742
- }
743
- this.audioSource = null;
744
780
  this.audioSender = null;
745
- this.analyser = null;
746
781
  this.isResponseInProgress = false;
747
782
  this.cancelInFlight = false;
748
783
  this.connectionLossNotified = false;
@@ -790,14 +825,10 @@ var OpenAIRealtimeClient = class {
790
825
  session: config ?? this.currentSessionConfig ?? {},
791
826
  sdpOffer: offer.sdp
792
827
  });
793
- if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
794
- throw new Error(
795
- `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
796
- );
797
- }
828
+ const answerSdp = this.resolveWebRTCAnswerSdp(bootstrap);
798
829
  await pc.setRemoteDescription({
799
830
  type: "answer",
800
- sdp: bootstrap.answerSdp
831
+ sdp: answerSdp
801
832
  });
802
833
  return true;
803
834
  }
@@ -835,11 +866,38 @@ var OpenAIRealtimeClient = class {
835
866
  getSessionUpdateTimeoutMs() {
836
867
  return this.options.sessionUpdateTimeoutMs ?? DEFAULT_SESSION_UPDATE_TIMEOUT_MS;
837
868
  }
869
+ resolveWebRTCAnswerSdp(bootstrap) {
870
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
871
+ throw new Error(
872
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
873
+ );
874
+ }
875
+ return bootstrap.answerSdp;
876
+ }
838
877
  resetResponseTracking() {
839
878
  this.assistantText = "";
840
879
  this.hasStartedAssistantResponse = false;
841
880
  this.hasStartedAudioOutput = false;
842
881
  }
882
+ ensureAssistantResponseStarted() {
883
+ if (!this.hasStartedAssistantResponse) {
884
+ this.hasStartedAssistantResponse = true;
885
+ this.emitEvent({ type: "assistant.response.started" });
886
+ }
887
+ }
888
+ emitFinalAssistantText(full) {
889
+ this.ensureAssistantResponseStarted();
890
+ if (full !== this.assistantText) {
891
+ const delta = full.startsWith(this.assistantText) ? full.slice(this.assistantText.length) : full;
892
+ this.assistantText = full;
893
+ if (delta) {
894
+ this.emitEvent({
895
+ type: "assistant.text.delta",
896
+ text: delta
897
+ });
898
+ }
899
+ }
900
+ }
843
901
  emitEvent(event) {
844
902
  for (const callback of this.eventCallbacks) {
845
903
  callback(event);
@@ -892,11 +950,6 @@ function toOpenAIRealtimeSessionUpdate(config) {
892
950
  }
893
951
  return session;
894
952
  }
895
- function delay(ms) {
896
- return new Promise((resolve) => {
897
- setTimeout(resolve, ms);
898
- });
899
- }
900
953
 
901
954
  exports.OpenAIRealtimeClient = OpenAIRealtimeClient;
902
955
  exports.createOpenAIRealtimeClient = createOpenAIRealtimeClient;