@charivo/realtime 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +23 -0
  2. package/dist/{chunk-FABRM56W.mjs → chunk-3O3COMUY.mjs} +216 -17
  3. package/dist/chunk-3O3COMUY.mjs.map +1 -0
  4. package/dist/{chunk-WLTIMNHQ.mjs → chunk-J5L3BPMG.mjs} +248 -13
  5. package/dist/chunk-J5L3BPMG.mjs.map +1 -0
  6. package/dist/chunk-ZJZO4QYA.mjs +123 -0
  7. package/dist/chunk-ZJZO4QYA.mjs.map +1 -0
  8. package/dist/index.d.mts +15 -3
  9. package/dist/index.d.ts +15 -3
  10. package/dist/index.js +240 -21
  11. package/dist/index.js.map +1 -1
  12. package/dist/index.mjs +237 -21
  13. package/dist/index.mjs.map +1 -1
  14. package/dist/openai/index.d.mts +29 -1
  15. package/dist/openai/index.d.ts +29 -1
  16. package/dist/openai/index.js +326 -13
  17. package/dist/openai/index.js.map +1 -1
  18. package/dist/openai/index.mjs +2 -2
  19. package/dist/openai-agents/index.d.mts +24 -1
  20. package/dist/openai-agents/index.d.ts +24 -1
  21. package/dist/openai-agents/index.js +293 -17
  22. package/dist/openai-agents/index.js.map +1 -1
  23. package/dist/openai-agents/index.mjs +2 -2
  24. package/dist/remote/index.d.mts +2 -1
  25. package/dist/remote/index.d.ts +2 -1
  26. package/dist/remote/index.js +572 -54
  27. package/dist/remote/index.js.map +1 -1
  28. package/dist/remote/index.mjs +38 -31
  29. package/dist/remote/index.mjs.map +1 -1
  30. package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
  31. package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
  32. package/package.json +2 -2
  33. package/dist/chunk-FABRM56W.mjs.map +0 -1
  34. package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
  35. package/dist/chunk-XJNT4ZTF.mjs +0 -41
  36. package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/index.mjs CHANGED
@@ -1,5 +1,6 @@
1
1
  import { SET_EXPRESSION_TOOL_DESCRIPTION, PLAY_MOTION_TOOL_DESCRIPTION, LOOK_AT_TOOL_DESCRIPTION, buildRealtimeSessionConfig } from './chunk-PDODY2T7.mjs';
2
2
  export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-PDODY2T7.mjs';
3
+ import { CharivoStateError, toCharivoError, subscribeBrowserLifecycle } from '@charivo/core';
3
4
 
4
5
  // src/tools.ts
5
6
  var MIN_GAZE = -1;
@@ -140,6 +141,7 @@ function clamp(value, min, max) {
140
141
 
141
142
  // src/realtime-manager.ts
142
143
  var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
144
+ var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
143
145
  var RealtimeManagerImpl = class {
144
146
  constructor(client, options = {}) {
145
147
  this.client = client;
@@ -147,6 +149,7 @@ var RealtimeManagerImpl = class {
147
149
  for (const tool of options.tools ?? []) {
148
150
  this.registerTool(tool);
149
151
  }
152
+ this.installBrowserLifecycleListeners();
150
153
  this.client.onEvent((event) => {
151
154
  void this.handleClientEvent(event).catch((error) => {
152
155
  this.applyError(
@@ -160,11 +163,16 @@ var RealtimeManagerImpl = class {
160
163
  character = null;
161
164
  isStoppingSession = false;
162
165
  isRefreshingSession = false;
166
+ isRecoveringConnection = false;
163
167
  isAudioPlaybackActive = false;
164
168
  sessionBaseConfig;
165
169
  refreshInFlight = null;
170
+ reconnectInFlight = null;
166
171
  hasQueuedRefresh = false;
167
172
  stopRequestedDuringRefresh = false;
173
+ reconnectToken = 0;
174
+ wakeLockSentinel = null;
175
+ teardownBrowserLifecycle;
168
176
  toolRegistry = /* @__PURE__ */ new Map();
169
177
  state = {
170
178
  connection: "idle",
@@ -224,8 +232,8 @@ var RealtimeManagerImpl = class {
224
232
  };
225
233
  }
226
234
  async startSession(config) {
227
- if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight) {
228
- throw new Error("Realtime session already active");
235
+ if (this.state.session.status === "active" || this.state.session.status === "starting" || this.refreshInFlight || this.reconnectInFlight) {
236
+ throw new CharivoStateError("Realtime session already active");
229
237
  }
230
238
  this.sessionBaseConfig = this.resolveNextSessionBaseConfig(config);
231
239
  const sessionConfig = this.buildEffectiveSessionConfig();
@@ -243,16 +251,17 @@ var RealtimeManagerImpl = class {
243
251
  },
244
252
  lastError: null
245
253
  };
254
+ this.installBrowserLifecycleListeners();
246
255
  this.emitState();
247
256
  try {
248
257
  await this.client.connect(sessionConfig);
249
258
  this.commitActiveSession(sessionConfig);
259
+ void this.requestWakeLock();
250
260
  this.emitSessionStart("user");
251
261
  } catch (error) {
252
- this.finalizeFailedSession(
253
- error instanceof Error ? error : new Error(String(error))
254
- );
255
- throw error;
262
+ const typedError = toCharivoError("transport", error);
263
+ this.finalizeFailedSession(typedError);
264
+ throw typedError;
256
265
  }
257
266
  }
258
267
  updateSession(config) {
@@ -260,6 +269,9 @@ var RealtimeManagerImpl = class {
260
269
  if (this.state.session.status !== "active") {
261
270
  return Promise.resolve();
262
271
  }
272
+ if (this.isRecoveringConnection) {
273
+ return Promise.resolve();
274
+ }
263
275
  if (this.refreshInFlight) {
264
276
  this.hasQueuedRefresh = true;
265
277
  return this.refreshInFlight;
@@ -274,6 +286,14 @@ var RealtimeManagerImpl = class {
274
286
  return this.refreshInFlight;
275
287
  }
276
288
  async stopSession() {
289
+ const reconnectInFlight = this.reconnectInFlight;
290
+ this.cancelReconnectLoop();
291
+ if (reconnectInFlight) {
292
+ try {
293
+ await reconnectInFlight;
294
+ } catch {
295
+ }
296
+ }
277
297
  if (this.refreshInFlight) {
278
298
  this.stopRequestedDuringRefresh = true;
279
299
  this.hasQueuedRefresh = false;
@@ -282,32 +302,45 @@ var RealtimeManagerImpl = class {
282
302
  } catch {
283
303
  }
284
304
  if (this.state.session.status === "active") {
285
- await this.performStopSession();
305
+ await this.performStopSession().catch((error) => {
306
+ throw toCharivoError("transport", error);
307
+ });
286
308
  }
287
309
  return;
288
310
  }
289
311
  if (this.state.session.status !== "active") {
290
312
  return;
291
313
  }
292
- await this.performStopSession();
314
+ await this.performStopSession().catch((error) => {
315
+ throw toCharivoError("transport", error);
316
+ });
293
317
  }
294
318
  async sendMessage(text) {
295
319
  if (this.state.session.status !== "active") {
296
- throw new Error("Realtime session not active");
320
+ throw new CharivoStateError("Realtime session not active");
321
+ }
322
+ if (this.state.connection !== "connected") {
323
+ throw new CharivoStateError("Realtime session is reconnecting");
297
324
  }
298
- await this.client.sendText(text);
325
+ await this.client.sendText(text).catch((error) => Promise.reject(toCharivoError("transport", error)));
299
326
  }
300
327
  async sendAudioChunk(audio) {
301
328
  if (this.state.session.status !== "active") {
302
- throw new Error("Realtime session not active");
329
+ throw new CharivoStateError("Realtime session not active");
330
+ }
331
+ if (this.state.connection !== "connected") {
332
+ throw new CharivoStateError("Realtime session is reconnecting");
303
333
  }
304
- await this.client.sendAudio(audio);
334
+ await this.client.sendAudio(audio).catch((error) => Promise.reject(toCharivoError("transport", error)));
305
335
  }
306
336
  async interrupt() {
307
337
  if (this.state.session.status !== "active") {
308
- throw new Error("Realtime session not active");
338
+ throw new CharivoStateError("Realtime session not active");
309
339
  }
310
- await this.client.interrupt();
340
+ if (this.state.connection !== "connected") {
341
+ throw new CharivoStateError("Realtime session is reconnecting");
342
+ }
343
+ await this.client.interrupt().catch((error) => Promise.reject(toCharivoError("transport", error)));
311
344
  this.state = {
312
345
  ...this.state,
313
346
  response: {
@@ -321,7 +354,7 @@ var RealtimeManagerImpl = class {
321
354
  async handleClientEvent(event) {
322
355
  switch (event.type) {
323
356
  case "session.started":
324
- if (this.state.connection === "connecting" || this.isRefreshingSession) {
357
+ if (this.state.connection === "connecting" || this.isRefreshingSession || this.isRecoveringConnection) {
325
358
  return;
326
359
  }
327
360
  this.state = mergeRealtimeState(this.state, {
@@ -335,11 +368,14 @@ var RealtimeManagerImpl = class {
335
368
  this.emitState();
336
369
  return;
337
370
  case "session.ended":
338
- if (this.isStoppingSession || this.isRefreshingSession || this.state.connection === "disconnecting") {
371
+ if (this.isStoppingSession || this.isRefreshingSession || this.isRecoveringConnection || this.state.connection === "disconnecting") {
339
372
  return;
340
373
  }
341
374
  this.finalizeStoppedSession(true, "user");
342
375
  return;
376
+ case "connection.lost":
377
+ this.beginReconnect(event.cause, event.error);
378
+ return;
343
379
  case "user.transcript":
344
380
  this.eventEmitter?.emit("realtime:user:transcript", {
345
381
  text: event.text
@@ -421,6 +457,14 @@ var RealtimeManagerImpl = class {
421
457
  this.emitState();
422
458
  return;
423
459
  case "error":
460
+ if (this.isRecoveringConnection && this.state.connection === "connecting") {
461
+ this.state = {
462
+ ...this.state,
463
+ lastError: event.error
464
+ };
465
+ this.emitState();
466
+ return;
467
+ }
424
468
  this.applyError(event.error, this.state.connection);
425
469
  return;
426
470
  }
@@ -544,19 +588,124 @@ var RealtimeManagerImpl = class {
544
588
  state: this.getState()
545
589
  });
546
590
  }
591
+ beginReconnect(cause, error) {
592
+ if (this.state.session.status !== "active" || this.isStoppingSession || this.state.connection === "disconnecting" || this.isRecoveringConnection) {
593
+ return;
594
+ }
595
+ this.isRecoveringConnection = true;
596
+ this.reconnectToken += 1;
597
+ const reconnectToken = this.reconnectToken;
598
+ if (this.state.response.status === "responding") {
599
+ this.emitAudioEnd();
600
+ this.state = {
601
+ ...this.state,
602
+ connection: "connecting",
603
+ response: {
604
+ status: "interrupted",
605
+ text: this.state.response.text
606
+ },
607
+ lastError: error ?? this.state.lastError
608
+ };
609
+ } else {
610
+ this.state = {
611
+ ...this.state,
612
+ connection: "connecting",
613
+ lastError: error ?? this.state.lastError
614
+ };
615
+ }
616
+ this.emitState();
617
+ this.reconnectInFlight = this.runReconnectLoop(reconnectToken, cause).catch((reconnectError) => {
618
+ this.finalizeFailedSession(reconnectError);
619
+ }).finally(() => {
620
+ if (this.reconnectToken === reconnectToken) {
621
+ this.reconnectInFlight = null;
622
+ this.isRecoveringConnection = false;
623
+ }
624
+ });
625
+ }
626
+ async runReconnectLoop(reconnectToken, cause) {
627
+ const startedAt = Date.now();
628
+ let lastError = new Error("Realtime reconnect exhausted");
629
+ for (let index = 0; index < RECONNECT_DELAYS_MS.length; index += 1) {
630
+ const attempt = index + 1;
631
+ const delayMs = RECONNECT_DELAYS_MS[index];
632
+ this.eventEmitter?.emit("realtime:reconnect:attempt", {
633
+ attempt,
634
+ delayMs,
635
+ cause
636
+ });
637
+ await delay(delayMs);
638
+ if (!this.isReconnectTokenActive(reconnectToken)) {
639
+ return;
640
+ }
641
+ const sessionConfig = this.buildEffectiveSessionConfig();
642
+ try {
643
+ await this.client.recover(sessionConfig);
644
+ if (!this.isReconnectTokenActive(reconnectToken)) {
645
+ return;
646
+ }
647
+ this.state = {
648
+ ...this.state,
649
+ connection: "connected",
650
+ session: {
651
+ ...this.state.session,
652
+ status: "active",
653
+ config: sessionConfig,
654
+ characterId: this.character?.id
655
+ },
656
+ lastError: null
657
+ };
658
+ this.emitState();
659
+ void this.requestWakeLock();
660
+ this.eventEmitter?.emit("realtime:reconnect:success", {
661
+ attempts: attempt,
662
+ totalMs: Date.now() - startedAt,
663
+ cause
664
+ });
665
+ return;
666
+ } catch (error) {
667
+ lastError = error instanceof Error ? error : new Error(String(error));
668
+ this.state = {
669
+ ...this.state,
670
+ connection: "connecting",
671
+ lastError
672
+ };
673
+ this.emitState();
674
+ }
675
+ }
676
+ if (!this.isReconnectTokenActive(reconnectToken)) {
677
+ return;
678
+ }
679
+ this.eventEmitter?.emit("realtime:reconnect:exhausted", {
680
+ attempts: RECONNECT_DELAYS_MS.length,
681
+ totalMs: Date.now() - startedAt,
682
+ cause,
683
+ lastError
684
+ });
685
+ throw lastError;
686
+ }
687
+ cancelReconnectLoop() {
688
+ this.reconnectToken += 1;
689
+ this.isRecoveringConnection = false;
690
+ this.reconnectInFlight = null;
691
+ }
692
+ isReconnectTokenActive(reconnectToken) {
693
+ return this.reconnectToken === reconnectToken && this.state.session.status === "active" && !this.isStoppingSession;
694
+ }
547
695
  emitState() {
548
696
  this.eventEmitter?.emit("realtime:state", {
549
697
  state: this.getState()
550
698
  });
551
699
  }
552
700
  applyError(error, connection) {
701
+ const typedError = toCharivoError("transport", error);
553
702
  this.emitAudioEnd();
554
703
  this.state = {
555
704
  ...this.state,
556
705
  connection,
557
- lastError: error
706
+ lastError: typedError
558
707
  };
559
- this.eventEmitter?.emit("realtime:error", { error });
708
+ this.eventEmitter?.emit("realtime:error", { error: typedError });
560
709
  this.emitState();
561
710
  }
562
711
  emitAudioStart() {
@@ -574,6 +723,56 @@ var RealtimeManagerImpl = class {
574
723
  this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
575
724
  this.eventEmitter?.emit("tts:audio:end", {});
576
725
  }
726
+ installBrowserLifecycleListeners() {
727
+ if (this.teardownBrowserLifecycle) {
728
+ return;
729
+ }
730
+ this.teardownBrowserLifecycle = subscribeBrowserLifecycle({
731
+ onHidden: () => {
732
+ void this.releaseWakeLock();
733
+ },
734
+ onPageHide: () => {
735
+ void this.releaseWakeLock();
736
+ },
737
+ onPageShow: () => {
738
+ void this.requestWakeLock();
739
+ },
740
+ onVisible: () => {
741
+ void this.requestWakeLock();
742
+ }
743
+ });
744
+ }
745
+ async requestWakeLock() {
746
+ if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
747
+ return;
748
+ }
749
+ if (this.wakeLockSentinel) {
750
+ return;
751
+ }
752
+ const wakeLockApi = navigator;
753
+ try {
754
+ this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
755
+ this.wakeLockSentinel?.addEventListener?.(
756
+ "release",
757
+ () => {
758
+ this.wakeLockSentinel = null;
759
+ },
760
+ { once: true }
761
+ );
762
+ } catch {
763
+ }
764
+ }
765
+ async releaseWakeLock() {
766
+ if (!this.wakeLockSentinel) {
767
+ return;
768
+ }
769
+ const wakeLockSentinel = this.wakeLockSentinel;
770
+ this.wakeLockSentinel = null;
771
+ try {
772
+ await wakeLockSentinel.release();
773
+ } catch {
774
+ }
775
+ }
577
776
  resolveNextSessionBaseConfig(config) {
578
777
  return mergeSessionConfig(
579
778
  this.sessionBaseConfig ?? this.options.defaultSessionConfig,
@@ -661,10 +860,14 @@ var RealtimeManagerImpl = class {
661
860
  this.commitPatchedSession(sessionConfig);
662
861
  } catch (error) {
663
862
  const refreshError = error instanceof Error ? error : new Error(String(error));
664
- if (this.state.lastError !== refreshError) {
665
- this.applyError(refreshError, "connected");
863
+ const typedRefreshError = toCharivoError("transport", refreshError);
864
+ if (this.isRecoveringConnection) {
865
+ throw typedRefreshError;
866
+ }
867
+ if (this.state.lastError !== typedRefreshError && this.state.lastError?.cause !== refreshError) {
868
+ this.applyError(typedRefreshError, "connected");
666
869
  }
667
- throw refreshError;
870
+ throw typedRefreshError;
668
871
  } finally {
669
872
  this.isRefreshingSession = false;
670
873
  }
@@ -682,6 +885,10 @@ var RealtimeManagerImpl = class {
682
885
  });
683
886
  }
684
887
  finalizeFailedSession(error) {
888
+ this.cancelReconnectLoop();
889
+ this.teardownBrowserLifecycle?.();
890
+ this.teardownBrowserLifecycle = void 0;
891
+ void this.releaseWakeLock();
685
892
  this.emitAudioEnd();
686
893
  this.state = {
687
894
  ...this.state,
@@ -701,6 +908,10 @@ var RealtimeManagerImpl = class {
701
908
  this.emitState();
702
909
  }
703
910
  finalizeStoppedSession(emitSessionEnd, reason) {
911
+ this.cancelReconnectLoop();
912
+ this.teardownBrowserLifecycle?.();
913
+ this.teardownBrowserLifecycle = void 0;
914
+ void this.releaseWakeLock();
704
915
  this.emitAudioEnd();
705
916
  this.state = {
706
917
  ...this.state,
@@ -788,6 +999,11 @@ async function withTimeout(promise, timeoutMs, toolName) {
788
999
  }
789
1000
  }
790
1001
  }
1002
+ function delay(ms) {
1003
+ return new Promise((resolve) => {
1004
+ setTimeout(resolve, ms);
1005
+ });
1006
+ }
791
1007
 
792
1008
  export { AVATAR_CONTROL_TOOL_NAMES, LOOK_AT_TOOL_NAME, PLAY_MOTION_TOOL_NAME, RealtimeManagerImpl, SET_EXPRESSION_TOOL_NAME, createAvatarControlTools, createRealtimeManager };
793
1009
  //# sourceMappingURL=index.mjs.map