@charivo/realtime 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-WLTIMNHQ.mjs → chunk-DHNRDHST.mjs} +248 -13
  3. package/dist/chunk-DHNRDHST.mjs.map +1 -0
  4. package/dist/{chunk-FABRM56W.mjs → chunk-PTFZ7CML.mjs} +208 -12
  5. package/dist/chunk-PTFZ7CML.mjs.map +1 -0
  6. package/dist/chunk-Y7JO2FFX.mjs +121 -0
  7. package/dist/chunk-Y7JO2FFX.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +217 -3
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +214 -3
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +323 -10
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +283 -9
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +531 -19
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +6 -3
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/index.d.ts CHANGED
@@ -1,7 +1,7 @@
1
- import { R as RealtimeTransportClient } from './types-Bc2tcYr1.js';
2
- export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-Bc2tcYr1.js';
1
+ import { R as RealtimeTransportClient } from './types-bmYuc_Vs.js';
2
+ export { b as RealtimeClient, a as RealtimeTransportEvent } from './types-bmYuc_Vs.js';
3
3
  import { RealtimeSessionConfig, RealtimeToolRegistration, RealtimeManager, CharivoEventEmitter, Character, RealtimeTool, RealtimeState, MotionSelection, GazeCoordinates, AvatarControlCatalog } from '@charivo/core';
4
- export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind } from '@charivo/core';
4
+ export { Character, CharivoEventEmitter, RealtimeConnectionState, RealtimeManager, RealtimeProvider, RealtimeReconnectCause, RealtimeResponseStatus, RealtimeSessionBootstrap, RealtimeSessionConfig, RealtimeSessionRequest, RealtimeSessionStatus, RealtimeSessionTransitionReason, RealtimeState, RealtimeTool, RealtimeToolChoice, RealtimeToolContext, RealtimeToolHandler, RealtimeToolRegistration, RealtimeTransportKind } from '@charivo/core';
5
5
 
6
6
  interface RealtimeManagerOptions {
7
7
  defaultSessionConfig?: Omit<RealtimeSessionConfig, "tools">;
@@ -22,11 +22,16 @@ declare class RealtimeManagerImpl implements RealtimeManager {
22
22
  private character;
23
23
  private isStoppingSession;
24
24
  private isRefreshingSession;
25
+ private isRecoveringConnection;
25
26
  private isAudioPlaybackActive;
26
27
  private sessionBaseConfig?;
27
28
  private refreshInFlight;
29
+ private reconnectInFlight;
28
30
  private hasQueuedRefresh;
29
31
  private stopRequestedDuringRefresh;
32
+ private reconnectToken;
33
+ private wakeLockSentinel;
34
+ private teardownBrowserLifecycle?;
30
35
  private readonly toolRegistry;
31
36
  private state;
32
37
  constructor(client: RealtimeTransportClient, options?: RealtimeManagerOptions);
@@ -49,10 +54,17 @@ declare class RealtimeManagerImpl implements RealtimeManager {
49
54
  private postProcessToolResult;
50
55
  private emitToolError;
51
56
  private ensureResponseStarted;
57
+ private beginReconnect;
58
+ private runReconnectLoop;
59
+ private cancelReconnectLoop;
60
+ private isReconnectTokenActive;
52
61
  private emitState;
53
62
  private applyError;
54
63
  private emitAudioStart;
55
64
  private emitAudioEnd;
65
+ private installBrowserLifecycleListeners;
66
+ private requestWakeLock;
67
+ private releaseWakeLock;
56
68
  private resolveNextSessionBaseConfig;
57
69
  private buildEffectiveSessionConfig;
58
70
  private commitActiveSession;
package/dist/index.js CHANGED
@@ -1,5 +1,9 @@
1
1
  'use strict';
2
2
 
3
+ var core = require('@charivo/core');
4
+
5
+ // src/realtime-manager.ts
6
+
3
7
  // src/instructions.ts
4
8
  var DEFAULT_PROVIDER = "openai";
5
9
  var DEFAULT_MODEL = "gpt-realtime-mini";
@@ -201,6 +205,7 @@ function clamp(value, min, max) {
201
205
 
202
206
  // src/realtime-manager.ts
203
207
  var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
208
+ var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
204
209
  var RealtimeManagerImpl = class {
205
210
  constructor(client, options = {}) {
206
211
  this.client = client;
@@ -208,6 +213,7 @@ var RealtimeManagerImpl = class {
208
213
  for (const tool of options.tools ?? []) {
209
214
  this.registerTool(tool);
210
215
  }
216
+ this.installBrowserLifecycleListeners();
211
217
  this.client.onEvent((event) => {
212
218
  void this.handleClientEvent(event).catch((error) => {
213
219
  this.applyError(
@@ -221,11 +227,16 @@ var RealtimeManagerImpl = class {
221
227
  character = null;
222
228
  isStoppingSession = false;
223
229
  isRefreshingSession = false;
230
+ isRecoveringConnection = false;
224
231
  isAudioPlaybackActive = false;
225
232
  sessionBaseConfig;
226
233
  refreshInFlight = null;
234
+ reconnectInFlight = null;
227
235
  hasQueuedRefresh = false;
228
236
  stopRequestedDuringRefresh = false;
237
+ reconnectToken = 0;
238
+ wakeLockSentinel = null;
239
+ teardownBrowserLifecycle;
229
240
  toolRegistry = /* @__PURE__ */ new Map();
230
241
  state = {
231
242
  connection: "idle",
@@ -285,7 +296,7 @@ var RealtimeManagerImpl = class {
285
296
  };
286
297
  }
287
298
  async startSession(config) {
288
- if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight) {
299
+ if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight || this.reconnectInFlight) {
289
300
  throw new Error("Realtime session already active");
290
301
  }
291
302
  this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
@@ -304,10 +315,12 @@ var RealtimeManagerImpl = class {
304
315
  },
305
316
  lastError: null
306
317
  };
318
+ this.installBrowserLifecycleListeners();
307
319
  this.emitState();
308
320
  try {
309
321
  await this.client.connect(sessionConfig);
310
322
  this.commitActiveSession(sessionConfig);
323
+ void this.requestWakeLock();
311
324
  this.emitSessionStart("user");
312
325
  } catch (error) {
313
326
  this.finalizeFailedSession(
@@ -321,6 +334,9 @@ var RealtimeManagerImpl = class {
321
334
  if (this.state.session.status !== "active") {
322
335
  return Promise.resolve();
323
336
  }
337
+ if (this.isRecoveringConnection) {
338
+ return Promise.resolve();
339
+ }
324
340
  if (this.refreshInFlight) {
325
341
  this.hasQueuedRefresh = true;
326
342
  return this.refreshInFlight;
@@ -335,6 +351,14 @@ var RealtimeManagerImpl = class {
335
351
  return this.refreshInFlight;
336
352
  }
337
353
  async stopSession() {
354
+ const reconnectInFlight = this.reconnectInFlight;
355
+ this.cancelReconnectLoop();
356
+ if (reconnectInFlight) {
357
+ try {
358
+ await reconnectInFlight;
359
+ } catch {
360
+ }
361
+ }
338
362
  if (this.refreshInFlight) {
339
363
  this.stopRequestedDuringRefresh = true;
340
364
  this.hasQueuedRefresh = false;
@@ -356,18 +380,27 @@ var RealtimeManagerImpl = class {
356
380
  if (this.state.session.status !== "active") {
357
381
  throw new Error("Realtime session not active");
358
382
  }
383
+ if (this.state.connection !== "connected") {
384
+ throw new Error("Realtime session is reconnecting");
385
+ }
359
386
  await this.client.sendText(text);
360
387
  }
361
388
  async sendAudioChunk(audio) {
362
389
  if (this.state.session.status !== "active") {
363
390
  throw new Error("Realtime session not active");
364
391
  }
392
+ if (this.state.connection !== "connected") {
393
+ throw new Error("Realtime session is reconnecting");
394
+ }
365
395
  await this.client.sendAudio(audio);
366
396
  }
367
397
  async interrupt() {
368
398
  if (this.state.session.status !== "active") {
369
399
  throw new Error("Realtime session not active");
370
400
  }
401
+ if (this.state.connection !== "connected") {
402
+ throw new Error("Realtime session is reconnecting");
403
+ }
371
404
  await this.client.interrupt();
372
405
  this.state = {
373
406
  ...this.state,
@@ -382,7 +415,7 @@ var RealtimeManagerImpl = class {
382
415
  async handleClientEvent(event) {
383
416
  switch (event.type) {
384
417
  case "session.started":
385
- if (this.state.connection === "connecting" || this.isRefreshingSession) {
418
+ if (this.state.connection === "connecting" || this.isRefreshingSession || this.isRecoveringConnection) {
386
419
  return;
387
420
  }
388
421
  this.state = mergeRealtimeState(this.state, {
@@ -396,11 +429,14 @@ var RealtimeManagerImpl = class {
396
429
  this.emitState();
397
430
  return;
398
431
  case "session.ended":
399
- if (this.isStoppingSession || this.isRefreshingSession || this.state.connection === "disconnecting") {
432
+ if (this.isStoppingSession || this.isRefreshingSession || this.isRecoveringConnection || this.state.connection === "disconnecting") {
400
433
  return;
401
434
  }
402
435
  this.finalizeStoppedSession(true, "user");
403
436
  return;
437
+ case "connection.lost":
438
+ this.beginReconnect(event.cause, event.error);
439
+ return;
404
440
  case "user.transcript":
405
441
  this.eventEmitter?.emit("realtime:user:transcript", {
406
442
  text: event.text
@@ -482,6 +518,14 @@ var RealtimeManagerImpl = class {
482
518
  this.emitState();
483
519
  return;
484
520
  case "error":
521
+ if (this.isRecoveringConnection && this.state.connection === "connecting") {
522
+ this.state = {
523
+ ...this.state,
524
+ lastError: event.error
525
+ };
526
+ this.emitState();
527
+ return;
528
+ }
485
529
  this.applyError(event.error, this.state.connection);
486
530
  return;
487
531
  }
@@ -605,6 +649,110 @@ var RealtimeManagerImpl = class {
605
649
  state: this.getState()
606
650
  });
607
651
  }
652
+ beginReconnect(cause, error) {
653
+ if (this.state.session.status !== "active" || this.isStoppingSession || this.state.connection === "disconnecting" || this.isRecoveringConnection) {
654
+ return;
655
+ }
656
+ this.isRecoveringConnection = true;
657
+ this.reconnectToken += 1;
658
+ const reconnectToken = this.reconnectToken;
659
+ if (this.state.response.status === "responding") {
660
+ this.emitAudioEnd();
661
+ this.state = {
662
+ ...this.state,
663
+ connection: "connecting",
664
+ response: {
665
+ status: "interrupted",
666
+ text: this.state.response.text
667
+ },
668
+ lastError: error ?? this.state.lastError
669
+ };
670
+ } else {
671
+ this.state = {
672
+ ...this.state,
673
+ connection: "connecting",
674
+ lastError: error ?? this.state.lastError
675
+ };
676
+ }
677
+ this.emitState();
678
+ this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
679
+ this.finalizeFailedSession(reconnectError);
680
+ }).finally(() => {
681
+ if (this.reconnectToken === reconnectToken) {
682
+ this.reconnectInFlight = null;
683
+ this.isRecoveringConnection = false;
684
+ }
685
+ });
686
+ }
687
+ async runReconnectLoop(reconnectToken, cause) {
688
+ const startedAt = Date.now();
689
+ let lastError = new Error("Realtime reconnect exhausted");
690
+ for (let index = 0; index < RECONNECT_DELAYS_MS.length; index += 1) {
691
+ const attempt = index + 1;
692
+ const delayMs = RECONNECT_DELAYS_MS[index];
693
+ this.eventEmitter?.emit("realtime:reconnect:attempt", {
694
+ attempt,
695
+ delayMs,
696
+ cause
697
+ });
698
+ await delay(delayMs);
699
+ if (!this.isReconnectTokenActive(reconnectToken)) {
700
+ return;
701
+ }
702
+ const sessionConfig = this.buildEffectiveSessionConfig();
703
+ try {
704
+ await this.client.recover(sessionConfig);
705
+ if (!this.isReconnectTokenActive(reconnectToken)) {
706
+ return;
707
+ }
708
+ this.state = {
709
+ ...this.state,
710
+ connection: "connected",
711
+ session: {
712
+ ...this.state.session,
713
+ status: "active",
714
+ config: sessionConfig,
715
+ characterId: this.character?.id
716
+ },
717
+ lastError: null
718
+ };
719
+ this.emitState();
720
+ void this.requestWakeLock();
721
+ this.eventEmitter?.emit("realtime:reconnect:success", {
722
+ attempts: attempt,
723
+ totalMs: Date.now() - startedAt,
724
+ cause
725
+ });
726
+ return;
727
+ } catch (error) {
728
+ lastError = error instanceof Error ? error : new Error(String(error));
729
+ this.state = {
730
+ ...this.state,
731
+ connection: "connecting",
732
+ lastError
733
+ };
734
+ this.emitState();
735
+ }
736
+ }
737
+ if (!this.isReconnectTokenActive(reconnectToken)) {
738
+ return;
739
+ }
740
+ this.eventEmitter?.emit("realtime:reconnect:exhausted", {
741
+ attempts: RECONNECT_DELAYS_MS.length,
742
+ totalMs: Date.now() - startedAt,
743
+ cause,
744
+ lastError
745
+ });
746
+ throw lastError;
747
+ }
748
+ cancelReconnectLoop() {
749
+ this.reconnectToken += 1;
750
+ this.isRecoveringConnection = false;
751
+ this.reconnectInFlight = null;
752
+ }
753
+ isReconnectTokenActive(reconnectToken) {
754
+ return this.reconnectToken === reconnectToken && this.state.session.status === "active" && !this.isStoppingSession;
755
+ }
608
756
  emitState() {
609
757
  this.eventEmitter?.emit("realtime:state", {
610
758
  state: this.getState()
@@ -635,6 +783,56 @@ var RealtimeManagerImpl = class {
635
783
  this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
636
784
  this.eventEmitter?.emit("tts:audio:end", {});
637
785
  }
786
+ installBrowserLifecycleListeners() {
787
+ if (this.teardownBrowserLifecycle) {
788
+ return;
789
+ }
790
+ this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
791
+ onHidden: () => {
792
+ void this.releaseWakeLock();
793
+ },
794
+ onPageHide: () => {
795
+ void this.releaseWakeLock();
796
+ },
797
+ onPageShow: () => {
798
+ void this.requestWakeLock();
799
+ },
800
+ onVisible: () => {
801
+ void this.requestWakeLock();
802
+ }
803
+ });
804
+ }
805
+ async requestWakeLock() {
806
+ if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
807
+ return;
808
+ }
809
+ if (this.wakeLockSentinel) {
810
+ return;
811
+ }
812
+ const wakeLockApi = navigator;
813
+ try {
814
+ this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
815
+ this.wakeLockSentinel?.addEventListener?.(
816
+ "release",
817
+ () => {
818
+ this.wakeLockSentinel = null;
819
+ },
820
+ { once: true }
821
+ );
822
+ } catch {
823
+ }
824
+ }
825
+ async releaseWakeLock() {
826
+ if (!this.wakeLockSentinel) {
827
+ return;
828
+ }
829
+ const wakeLockSentinel = this.wakeLockSentinel;
830
+ this.wakeLockSentinel = null;
831
+ try {
832
+ await wakeLockSentinel.release();
833
+ } catch {
834
+ }
835
+ }
638
836
  resolveNextSessionBaseConfig(config) {
639
837
  return mergeSessionConfig(
640
838
  this.sessionBaseConfig ?? this.options.defaultSessionConfig,
@@ -722,6 +920,9 @@ var RealtimeManagerImpl = class {
722
920
  this.commitPatchedSession(sessionConfig);
723
921
  } catch (error) {
724
922
  const refreshError = error instanceof Error ? error : new Error(String(error));
923
+ if (this.isRecoveringConnection) {
924
+ throw refreshError;
925
+ }
725
926
  if (this.state.lastError !== refreshError) {
726
927
  this.applyError(refreshError, "connected");
727
928
  }
@@ -743,6 +944,10 @@ var RealtimeManagerImpl = class {
743
944
  });
744
945
  }
745
946
  finalizeFailedSession(error) {
947
+ this.cancelReconnectLoop();
948
+ this.teardownBrowserLifecycle?.();
949
+ this.teardownBrowserLifecycle = void 0;
950
+ void this.releaseWakeLock();
746
951
  this.emitAudioEnd();
747
952
  this.state = {
748
953
  ...this.state,
@@ -762,6 +967,10 @@ var RealtimeManagerImpl = class {
762
967
  this.emitState();
763
968
  }
764
969
  finalizeStoppedSession(emitSessionEnd, reason) {
970
+ this.cancelReconnectLoop();
971
+ this.teardownBrowserLifecycle?.();
972
+ this.teardownBrowserLifecycle = void 0;
973
+ void this.releaseWakeLock();
765
974
  this.emitAudioEnd();
766
975
  this.state = {
767
976
  ...this.state,
@@ -849,6 +1058,11 @@ async function withTimeout(promise, timeoutMs, toolName) {
849
1058
  }
850
1059
  }
851
1060
  }
1061
+ function delay(ms) {
1062
+ return new Promise((resolve) => {
1063
+ setTimeout(resolve, ms);
1064
+ });
1065
+ }
852
1066
 
853
1067
  exports.AVATAR_CONTROL_TOOL_NAMES = AVATAR_CONTROL_TOOL_NAMES;
854
1068
  exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;