@charivo/realtime 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +32 -7
  2. package/dist/{chunk-IIKECRXN.mjs → chunk-DHNRDHST.mjs} +350 -17
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/chunk-PDODY2T7.mjs +65 -0
  5. package/dist/chunk-PDODY2T7.mjs.map +1 -0
  6. package/dist/{chunk-JGJZKKMH.mjs → chunk-PTFZ7CML.mjs} +235 -22
  7. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  8. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  9. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  10. package/dist/index.d.mts +16 -4
  11. package/dist/index.d.ts +16 -4
  12. package/dist/index.js +240 -38
  13. package/dist/index.js.map +1 -1
  14. package/dist/index.mjs +238 -98
  15. package/dist/index.mjs.map +1 -1
  16. package/dist/openai/index.d.mts +37 -1
  17. package/dist/openai/index.d.ts +37 -1
  18. package/dist/openai/index.js +427 -14
  19. package/dist/openai/index.js.map +1 -1
  20. package/dist/openai/index.mjs +3 -2
  21. package/dist/openai-agents/index.d.mts +26 -1
  22. package/dist/openai-agents/index.d.ts +26 -1
  23. package/dist/openai-agents/index.js +311 -18
  24. package/dist/openai-agents/index.js.map +1 -1
  25. package/dist/openai-agents/index.mjs +3 -2
  26. package/dist/remote/index.d.mts +3 -1
  27. package/dist/remote/index.d.ts +3 -1
  28. package/dist/remote/index.js +663 -32
  29. package/dist/remote/index.js.map +1 -1
  30. package/dist/remote/index.mjs +10 -3
  31. package/dist/remote/index.mjs.map +1 -1
  32. package/dist/{types-DntfJo0o.d.mts → types-bmYuc_Vs.d.mts} +7 -1
  33. package/dist/{types-DntfJo0o.d.ts → types-bmYuc_Vs.d.ts} +7 -1
  34. package/package.json +3 -3
  35. package/dist/chunk-IIKECRXN.mjs.map +0 -1
  36. package/dist/chunk-JGJZKKMH.mjs.map +0 -1
  37. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  38. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/index.js CHANGED
@@ -1,9 +1,13 @@
1
1
  'use strict';
2
2
 
3
+ var core = require('@charivo/core');
4
+
5
+ // src/realtime-manager.ts
6
+
3
7
  // src/instructions.ts
4
8
  var DEFAULT_PROVIDER = "openai";
5
9
  var DEFAULT_MODEL = "gpt-realtime-mini";
6
- var DEFAULT_VOICE = "marin";
10
+ var DEFAULT_REALTIME_VOICE = "marin";
7
11
  var SET_EXPRESSION_TOOL_DESCRIPTION = "Change the avatar's facial expression only when the emotional beat clearly shifts or should linger across the reply. Do not use this for every polite or lightweight reaction.";
8
12
  var PLAY_MOTION_TOOL_DESCRIPTION = "Play a noticeable body motion for greetings, emphasis, or bigger reaction beats. Prefer this over stacking multiple smaller actions when the moment needs one clear accent. Usually use at most one motion in a reply.";
9
13
  var LOOK_AT_TOOL_DESCRIPTION = `Shift the avatar's gaze for subtle attention changes or conversational focus. Trigger this on natural phrases like "glance", "look over", "peek at", or directional cues. Prefer this before "setExpression" when a lightweight reaction is enough.`;
@@ -29,7 +33,7 @@ function buildRealtimeSessionConfig({
29
33
  provider: baseConfig?.provider ?? DEFAULT_PROVIDER,
30
34
  transport: baseConfig?.transport ?? "webrtc",
31
35
  model: baseConfig?.model ?? DEFAULT_MODEL,
32
- voice: baseConfig?.voice ?? character?.voice?.voiceId ?? DEFAULT_VOICE,
36
+ voice: baseConfig?.voice ?? character?.voice?.voiceId ?? DEFAULT_REALTIME_VOICE,
33
37
  instructions: baseConfig?.instructions ?? buildCharacterInstructions(character),
34
38
  temperature: baseConfig?.temperature,
35
39
  maxTokens: baseConfig?.maxTokens,
@@ -201,6 +205,7 @@ function clamp(value, min, max) {
201
205
 
202
206
  // src/realtime-manager.ts
203
207
  var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
208
+ var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
204
209
  var RealtimeManagerImpl = class {
205
210
  constructor(client, options = {}) {
206
211
  this.client = client;
@@ -208,6 +213,7 @@ var RealtimeManagerImpl = class {
208
213
  for (const tool of options.tools ?? []) {
209
214
  this.registerTool(tool);
210
215
  }
216
+ this.installBrowserLifecycleListeners();
211
217
  this.client.onEvent((event) => {
212
218
  void this.handleClientEvent(event).catch((error) => {
213
219
  this.applyError(
@@ -221,12 +227,16 @@ var RealtimeManagerImpl = class {
221
227
  character = null;
222
228
  isStoppingSession = false;
223
229
  isRefreshingSession = false;
230
+ isRecoveringConnection = false;
224
231
  isAudioPlaybackActive = false;
225
232
  sessionBaseConfig;
226
233
  refreshInFlight = null;
234
+ reconnectInFlight = null;
227
235
  hasQueuedRefresh = false;
228
236
  stopRequestedDuringRefresh = false;
229
- emittedUserStopDuringRefresh = false;
237
+ reconnectToken = 0;
238
+ wakeLockSentinel = null;
239
+ teardownBrowserLifecycle;
230
240
  toolRegistry = /* @__PURE__ */ new Map();
231
241
  state = {
232
242
  connection: "idle",
@@ -286,7 +296,7 @@ var RealtimeManagerImpl = class {
286
296
  };
287
297
  }
288
298
  async startSession(config) {
289
- if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight) {
299
+ if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight || this.reconnectInFlight) {
290
300
  throw new Error("Realtime session already active");
291
301
  }
292
302
  this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
@@ -305,10 +315,12 @@ var RealtimeManagerImpl = class {
305
315
  },
306
316
  lastError: null
307
317
  };
318
+ this.installBrowserLifecycleListeners();
308
319
  this.emitState();
309
320
  try {
310
321
  await this.client.connect(sessionConfig);
311
322
  this.commitActiveSession(sessionConfig);
323
+ void this.requestWakeLock();
312
324
  this.emitSessionStart("user");
313
325
  } catch (error) {
314
326
  this.finalizeFailedSession(
@@ -322,22 +334,31 @@ var RealtimeManagerImpl = class {
322
334
  if (this.state.session.status !== "active") {
323
335
  return Promise.resolve();
324
336
  }
337
+ if (this.isRecoveringConnection) {
338
+ return Promise.resolve();
339
+ }
325
340
  if (this.refreshInFlight) {
326
341
  this.hasQueuedRefresh = true;
327
342
  return this.refreshInFlight;
328
343
  }
329
344
  this.stopRequestedDuringRefresh = false;
330
345
  this.hasQueuedRefresh = false;
331
- this.emittedUserStopDuringRefresh = false;
332
346
  this.refreshInFlight = this.runRefreshLoop().finally(() => {
333
347
  this.refreshInFlight = null;
334
348
  this.hasQueuedRefresh = false;
335
349
  this.stopRequestedDuringRefresh = false;
336
- this.emittedUserStopDuringRefresh = false;
337
350
  });
338
351
  return this.refreshInFlight;
339
352
  }
340
353
  async stopSession() {
354
+ const reconnectInFlight = this.reconnectInFlight;
355
+ this.cancelReconnectLoop();
356
+ if (reconnectInFlight) {
357
+ try {
358
+ await reconnectInFlight;
359
+ } catch {
360
+ }
361
+ }
341
362
  if (this.refreshInFlight) {
342
363
  this.stopRequestedDuringRefresh = true;
343
364
  this.hasQueuedRefresh = false;
@@ -347,8 +368,6 @@ var RealtimeManagerImpl = class {
347
368
  }
348
369
  if (this.state.session.status === "active") {
349
370
  await this.performStopSession();
350
- } else if (this.stopRequestedDuringRefresh && !this.emittedUserStopDuringRefresh) {
351
- this.emitSessionEnd("user");
352
371
  }
353
372
  return;
354
373
  }
@@ -361,18 +380,27 @@ var RealtimeManagerImpl = class {
361
380
  if (this.state.session.status !== "active") {
362
381
  throw new Error("Realtime session not active");
363
382
  }
383
+ if (this.state.connection !== "connected") {
384
+ throw new Error("Realtime session is reconnecting");
385
+ }
364
386
  await this.client.sendText(text);
365
387
  }
366
388
  async sendAudioChunk(audio) {
367
389
  if (this.state.session.status !== "active") {
368
390
  throw new Error("Realtime session not active");
369
391
  }
392
+ if (this.state.connection !== "connected") {
393
+ throw new Error("Realtime session is reconnecting");
394
+ }
370
395
  await this.client.sendAudio(audio);
371
396
  }
372
397
  async interrupt() {
373
398
  if (this.state.session.status !== "active") {
374
399
  throw new Error("Realtime session not active");
375
400
  }
401
+ if (this.state.connection !== "connected") {
402
+ throw new Error("Realtime session is reconnecting");
403
+ }
376
404
  await this.client.interrupt();
377
405
  this.state = {
378
406
  ...this.state,
@@ -387,7 +415,7 @@ var RealtimeManagerImpl = class {
387
415
  async handleClientEvent(event) {
388
416
  switch (event.type) {
389
417
  case "session.started":
390
- if (this.state.connection === "connecting" || this.isRefreshingSession) {
418
+ if (this.state.connection === "connecting" || this.isRefreshingSession || this.isRecoveringConnection) {
391
419
  return;
392
420
  }
393
421
  this.state = mergeRealtimeState(this.state, {
@@ -401,11 +429,14 @@ var RealtimeManagerImpl = class {
401
429
  this.emitState();
402
430
  return;
403
431
  case "session.ended":
404
- if (this.isStoppingSession || this.isRefreshingSession || this.state.connection === "disconnecting") {
432
+ if (this.isStoppingSession || this.isRefreshingSession || this.isRecoveringConnection || this.state.connection === "disconnecting") {
405
433
  return;
406
434
  }
407
435
  this.finalizeStoppedSession(true, "user");
408
436
  return;
437
+ case "connection.lost":
438
+ this.beginReconnect(event.cause, event.error);
439
+ return;
409
440
  case "user.transcript":
410
441
  this.eventEmitter?.emit("realtime:user:transcript", {
411
442
  text: event.text
@@ -487,6 +518,14 @@ var RealtimeManagerImpl = class {
487
518
  this.emitState();
488
519
  return;
489
520
  case "error":
521
+ if (this.isRecoveringConnection && this.state.connection === "connecting") {
522
+ this.state = {
523
+ ...this.state,
524
+ lastError: event.error
525
+ };
526
+ this.emitState();
527
+ return;
528
+ }
490
529
  this.applyError(event.error, this.state.connection);
491
530
  return;
492
531
  }
@@ -610,6 +649,110 @@ var RealtimeManagerImpl = class {
610
649
  state: this.getState()
611
650
  });
612
651
  }
652
+ beginReconnect(cause, error) {
653
+ if (this.state.session.status !== "active" || this.isStoppingSession || this.state.connection === "disconnecting" || this.isRecoveringConnection) {
654
+ return;
655
+ }
656
+ this.isRecoveringConnection = true;
657
+ this.reconnectToken += 1;
658
+ const reconnectToken = this.reconnectToken;
659
+ if (this.state.response.status === "responding") {
660
+ this.emitAudioEnd();
661
+ this.state = {
662
+ ...this.state,
663
+ connection: "connecting",
664
+ response: {
665
+ status: "interrupted",
666
+ text: this.state.response.text
667
+ },
668
+ lastError: error ?? this.state.lastError
669
+ };
670
+ } else {
671
+ this.state = {
672
+ ...this.state,
673
+ connection: "connecting",
674
+ lastError: error ?? this.state.lastError
675
+ };
676
+ }
677
+ this.emitState();
678
+ this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
679
+ this.finalizeFailedSession(reconnectError);
680
+ }).finally(() => {
681
+ if (this.reconnectToken === reconnectToken) {
682
+ this.reconnectInFlight = null;
683
+ this.isRecoveringConnection = false;
684
+ }
685
+ });
686
+ }
687
+ async runReconnectLoop(reconnectToken, cause) {
688
+ const startedAt = Date.now();
689
+ let lastError = new Error("Realtime reconnect exhausted");
690
+ for (let index = 0; index < RECONNECT_DELAYS_MS.length; index += 1) {
691
+ const attempt = index + 1;
692
+ const delayMs = RECONNECT_DELAYS_MS[index];
693
+ this.eventEmitter?.emit("realtime:reconnect:attempt", {
694
+ attempt,
695
+ delayMs,
696
+ cause
697
+ });
698
+ await delay(delayMs);
699
+ if (!this.isReconnectTokenActive(reconnectToken)) {
700
+ return;
701
+ }
702
+ const sessionConfig = this.buildEffectiveSessionConfig();
703
+ try {
704
+ await this.client.recover(sessionConfig);
705
+ if (!this.isReconnectTokenActive(reconnectToken)) {
706
+ return;
707
+ }
708
+ this.state = {
709
+ ...this.state,
710
+ connection: "connected",
711
+ session: {
712
+ ...this.state.session,
713
+ status: "active",
714
+ config: sessionConfig,
715
+ characterId: this.character?.id
716
+ },
717
+ lastError: null
718
+ };
719
+ this.emitState();
720
+ void this.requestWakeLock();
721
+ this.eventEmitter?.emit("realtime:reconnect:success", {
722
+ attempts: attempt,
723
+ totalMs: Date.now() - startedAt,
724
+ cause
725
+ });
726
+ return;
727
+ } catch (error) {
728
+ lastError = error instanceof Error ? error : new Error(String(error));
729
+ this.state = {
730
+ ...this.state,
731
+ connection: "connecting",
732
+ lastError
733
+ };
734
+ this.emitState();
735
+ }
736
+ }
737
+ if (!this.isReconnectTokenActive(reconnectToken)) {
738
+ return;
739
+ }
740
+ this.eventEmitter?.emit("realtime:reconnect:exhausted", {
741
+ attempts: RECONNECT_DELAYS_MS.length,
742
+ totalMs: Date.now() - startedAt,
743
+ cause,
744
+ lastError
745
+ });
746
+ throw lastError;
747
+ }
748
+ cancelReconnectLoop() {
749
+ this.reconnectToken += 1;
750
+ this.isRecoveringConnection = false;
751
+ this.reconnectInFlight = null;
752
+ }
753
+ isReconnectTokenActive(reconnectToken) {
754
+ return this.reconnectToken === reconnectToken && this.state.session.status === "active" && !this.isStoppingSession;
755
+ }
613
756
  emitState() {
614
757
  this.eventEmitter?.emit("realtime:state", {
615
758
  state: this.getState()
@@ -640,6 +783,56 @@ var RealtimeManagerImpl = class {
640
783
  this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
641
784
  this.eventEmitter?.emit("tts:audio:end", {});
642
785
  }
786
+ installBrowserLifecycleListeners() {
787
+ if (this.teardownBrowserLifecycle) {
788
+ return;
789
+ }
790
+ this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
791
+ onHidden: () => {
792
+ void this.releaseWakeLock();
793
+ },
794
+ onPageHide: () => {
795
+ void this.releaseWakeLock();
796
+ },
797
+ onPageShow: () => {
798
+ void this.requestWakeLock();
799
+ },
800
+ onVisible: () => {
801
+ void this.requestWakeLock();
802
+ }
803
+ });
804
+ }
805
+ async requestWakeLock() {
806
+ if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
807
+ return;
808
+ }
809
+ if (this.wakeLockSentinel) {
810
+ return;
811
+ }
812
+ const wakeLockApi = navigator;
813
+ try {
814
+ this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
815
+ this.wakeLockSentinel?.addEventListener?.(
816
+ "release",
817
+ () => {
818
+ this.wakeLockSentinel = null;
819
+ },
820
+ { once: true }
821
+ );
822
+ } catch {
823
+ }
824
+ }
825
+ async releaseWakeLock() {
826
+ if (!this.wakeLockSentinel) {
827
+ return;
828
+ }
829
+ const wakeLockSentinel = this.wakeLockSentinel;
830
+ this.wakeLockSentinel = null;
831
+ try {
832
+ await wakeLockSentinel.release();
833
+ } catch {
834
+ }
835
+ }
643
836
  resolveNextSessionBaseConfig(config) {
644
837
  return mergeSessionConfig(
645
838
  this.sessionBaseConfig ?? this.options.defaultSessionConfig,
@@ -672,6 +865,20 @@ var RealtimeManagerImpl = class {
672
865
  };
673
866
  this.emitState();
674
867
  }
868
+ commitPatchedSession(config) {
869
+ this.state = {
870
+ ...this.state,
871
+ connection: "connected",
872
+ session: {
873
+ ...this.state.session,
874
+ status: "active",
875
+ config,
876
+ characterId: this.character?.id
877
+ },
878
+ lastError: null
879
+ };
880
+ this.emitState();
881
+ }
675
882
  async performStopSession() {
676
883
  this.isStoppingSession = true;
677
884
  this.state = {
@@ -698,46 +905,28 @@ var RealtimeManagerImpl = class {
698
905
  } while (this.hasQueuedRefresh && !this.stopRequestedDuringRefresh);
699
906
  }
700
907
  async performSessionRefresh() {
701
- const previousState = this.getState();
702
908
  const sessionConfig = this.buildEffectiveSessionConfig();
703
909
  this.isRefreshingSession = true;
704
910
  this.state = {
705
911
  ...this.state,
706
- connection: "disconnecting",
707
912
  lastError: null
708
913
  };
709
914
  this.emitState();
710
915
  try {
711
- await this.client.disconnect();
916
+ await this.client.updateSession(sessionConfig);
712
917
  if (this.stopRequestedDuringRefresh) {
713
- this.emittedUserStopDuringRefresh = true;
714
- this.finalizeStoppedSession(true, "user");
715
918
  return;
716
919
  }
717
- this.state = {
718
- ...this.state,
719
- connection: "connecting",
720
- response: {
721
- status: "idle",
722
- text: ""
723
- }
724
- };
725
- this.emitState();
726
- await this.client.connect(sessionConfig);
727
- if (this.stopRequestedDuringRefresh) {
728
- await this.client.disconnect();
729
- this.emittedUserStopDuringRefresh = true;
730
- this.finalizeStoppedSession(true, "user");
731
- return;
732
- }
733
- this.commitActiveSession(sessionConfig);
734
- this.emitSessionEnd("refresh", previousState);
735
- this.emitSessionStart("refresh");
920
+ this.commitPatchedSession(sessionConfig);
736
921
  } catch (error) {
737
- this.finalizeFailedSession(
738
- error instanceof Error ? error : new Error(String(error))
739
- );
740
- throw error;
922
+ const refreshError = error instanceof Error ? error : new Error(String(error));
923
+ if (this.isRecoveringConnection) {
924
+ throw refreshError;
925
+ }
926
+ if (this.state.lastError !== refreshError) {
927
+ this.applyError(refreshError, "connected");
928
+ }
929
+ throw refreshError;
741
930
  } finally {
742
931
  this.isRefreshingSession = false;
743
932
  }
@@ -755,6 +944,10 @@ var RealtimeManagerImpl = class {
755
944
  });
756
945
  }
757
946
  finalizeFailedSession(error) {
947
+ this.cancelReconnectLoop();
948
+ this.teardownBrowserLifecycle?.();
949
+ this.teardownBrowserLifecycle = void 0;
950
+ void this.releaseWakeLock();
758
951
  this.emitAudioEnd();
759
952
  this.state = {
760
953
  ...this.state,
@@ -774,6 +967,10 @@ var RealtimeManagerImpl = class {
774
967
  this.emitState();
775
968
  }
776
969
  finalizeStoppedSession(emitSessionEnd, reason) {
970
+ this.cancelReconnectLoop();
971
+ this.teardownBrowserLifecycle?.();
972
+ this.teardownBrowserLifecycle = void 0;
973
+ void this.releaseWakeLock();
777
974
  this.emitAudioEnd();
778
975
  this.state = {
779
976
  ...this.state,
@@ -861,6 +1058,11 @@ async function withTimeout(promise, timeoutMs, toolName) {
861
1058
  }
862
1059
  }
863
1060
  }
1061
+ function delay(ms) {
1062
+ return new Promise((resolve) => {
1063
+ setTimeout(resolve, ms);
1064
+ });
1065
+ }
864
1066
 
865
1067
  exports.AVATAR_CONTROL_TOOL_NAMES = AVATAR_CONTROL_TOOL_NAMES;
866
1068
  exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;