@rapidmx/meet-plugin 0.6.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +18 -5
  2. package/apps/meet/[token].tsx +11 -10
  3. package/apps/meet/_CallControls.tsx +28 -2
  4. package/apps/meet/_CallView.tsx +26 -2
  5. package/apps/meet/_EffectsPanel.tsx +134 -0
  6. package/apps/meet/_MeetChrome.tsx +1 -1
  7. package/apps/meet/_MeetLobby.tsx +36 -4
  8. package/apps/meet/_ParticipantTile.tsx +73 -10
  9. package/apps/meet/_icons.tsx +4 -0
  10. package/apps/meet/_layout.tsx +2 -2
  11. package/apps/meet/_meetApi.ts +7 -1
  12. package/apps/meet/index.tsx +2 -2
  13. package/apps/settings-video-conferencing/_PersonalRoomCard.tsx +4 -4
  14. package/apps/settings-video-conferencing/_layout.tsx +2 -2
  15. package/apps/settings-video-conferencing/index.tsx +3 -3
  16. package/apps/shared/media/filters/VideoFilterProcessor.ts +408 -0
  17. package/apps/shared/media/filters/accessories.ts +262 -0
  18. package/apps/shared/media/filters/backgroundImage.ts +69 -0
  19. package/apps/shared/media/filters/filterTypes.ts +89 -0
  20. package/apps/shared/media/filters/mlModels.ts +174 -0
  21. package/apps/shared/media/filters/pixelEffects.ts +61 -0
  22. package/apps/shared/media/mediaPreferences.ts +105 -0
  23. package/apps/shared/media/useLocalMedia.ts +198 -14
  24. package/apps/shared/push/GuestSignalingClient.ts +4 -4
  25. package/apps/shared/relay/AudioPlayer.ts +111 -0
  26. package/apps/shared/relay/AudioSender.ts +194 -0
  27. package/apps/shared/relay/RelayClient.ts +262 -0
  28. package/apps/shared/relay/RelayReceiver.ts +72 -0
  29. package/apps/shared/relay/RelaySender.ts +74 -0
  30. package/apps/shared/relay/RelayTransport.ts +188 -0
  31. package/apps/shared/relay/VideoPlayer.ts +110 -0
  32. package/apps/shared/relay/VideoSender.ts +216 -0
  33. package/apps/shared/relay/audioResume.ts +55 -0
  34. package/apps/shared/relay/frames.ts +275 -0
  35. package/apps/shared/relay/relayEnv.ts +230 -0
  36. package/apps/shared/relay/senderCommon.ts +30 -0
  37. package/apps/shared/webrtc/MeshConnectionManager.ts +159 -7
  38. package/apps/shared/webrtc/realPeerConnection.ts +22 -0
  39. package/apps/shared/webrtc/types.ts +32 -2
  40. package/dist/apps/meet/[token].d.ts +3 -3
  41. package/dist/apps/meet/[token].js +10 -10
  42. package/dist/apps/meet/_CallControls.d.ts +2 -1
  43. package/dist/apps/meet/_CallControls.js +6 -3
  44. package/dist/apps/meet/_CallView.d.ts +4 -1
  45. package/dist/apps/meet/_CallView.js +20 -3
  46. package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
  47. package/dist/apps/meet/_EffectsPanel.js +60 -0
  48. package/dist/apps/meet/_MeetChrome.d.ts +1 -1
  49. package/dist/apps/meet/_MeetLobby.d.ts +8 -0
  50. package/dist/apps/meet/_MeetLobby.js +15 -4
  51. package/dist/apps/meet/_ParticipantTile.d.ts +17 -2
  52. package/dist/apps/meet/_ParticipantTile.js +27 -4
  53. package/dist/apps/meet/_icons.d.ts +1 -0
  54. package/dist/apps/meet/_icons.js +1 -0
  55. package/dist/apps/meet/_layout.d.ts +1 -1
  56. package/dist/apps/meet/_layout.js +1 -1
  57. package/dist/apps/meet/_meetApi.d.ts +6 -0
  58. package/dist/apps/meet/_meetApi.js +1 -1
  59. package/dist/apps/meet/index.js +2 -2
  60. package/dist/apps/settings-video-conferencing/_PersonalRoomCard.d.ts +1 -1
  61. package/dist/apps/settings-video-conferencing/_PersonalRoomCard.js +4 -4
  62. package/dist/apps/settings-video-conferencing/_layout.d.ts +1 -1
  63. package/dist/apps/settings-video-conferencing/_layout.js +1 -1
  64. package/dist/apps/settings-video-conferencing/index.d.ts +1 -1
  65. package/dist/apps/settings-video-conferencing/index.js +3 -3
  66. package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +78 -0
  67. package/dist/apps/shared/media/filters/VideoFilterProcessor.js +329 -0
  68. package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
  69. package/dist/apps/shared/media/filters/accessories.js +211 -0
  70. package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
  71. package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
  72. package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
  73. package/dist/apps/shared/media/filters/filterTypes.js +63 -0
  74. package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
  75. package/dist/apps/shared/media/filters/mlModels.js +106 -0
  76. package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
  77. package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
  78. package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
  79. package/dist/apps/shared/media/mediaPreferences.js +93 -0
  80. package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
  81. package/dist/apps/shared/media/useLocalMedia.js +160 -14
  82. package/dist/apps/shared/push/GuestSignalingClient.d.ts +1 -1
  83. package/dist/apps/shared/push/GuestSignalingClient.js +3 -3
  84. package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
  85. package/dist/apps/shared/relay/AudioPlayer.js +103 -0
  86. package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
  87. package/dist/apps/shared/relay/AudioSender.js +172 -0
  88. package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
  89. package/dist/apps/shared/relay/RelayClient.js +236 -0
  90. package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
  91. package/dist/apps/shared/relay/RelayReceiver.js +59 -0
  92. package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
  93. package/dist/apps/shared/relay/RelaySender.js +54 -0
  94. package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
  95. package/dist/apps/shared/relay/RelayTransport.js +142 -0
  96. package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
  97. package/dist/apps/shared/relay/VideoPlayer.js +102 -0
  98. package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
  99. package/dist/apps/shared/relay/VideoSender.js +204 -0
  100. package/dist/apps/shared/relay/audioResume.d.ts +12 -0
  101. package/dist/apps/shared/relay/audioResume.js +45 -0
  102. package/dist/apps/shared/relay/frames.d.ts +101 -0
  103. package/dist/apps/shared/relay/frames.js +221 -0
  104. package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
  105. package/dist/apps/shared/relay/relayEnv.js +69 -0
  106. package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
  107. package/dist/apps/shared/relay/senderCommon.js +10 -0
  108. package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +52 -2
  109. package/dist/apps/shared/webrtc/MeshConnectionManager.js +138 -7
  110. package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
  111. package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
  112. package/dist/apps/shared/webrtc/types.d.ts +29 -2
  113. package/dist/lib/index.js +2 -0
  114. package/dist/lib/index.js.map +1 -1
  115. package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
  116. package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
  117. package/dist/lib/util/EffectsUtils.js +21 -0
  118. package/dist/lib/util/EffectsUtils.js.map +1 -0
  119. package/dist/lib/util/IceServerUtils.js +25 -4
  120. package/dist/lib/util/IceServerUtils.js.map +1 -1
  121. package/dist/lib/util/RedisRelayBus.js +241 -0
  122. package/dist/lib/util/RedisRelayBus.js.map +1 -0
  123. package/dist/lib/util/RelayHub.js +405 -0
  124. package/dist/lib/util/RelayHub.js.map +1 -0
  125. package/dist/types/index.d.ts +2 -0
  126. package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
  127. package/dist/types/util/EffectsUtils.d.ts +10 -0
  128. package/dist/types/util/IceServerUtils.d.ts +12 -1
  129. package/dist/types/util/RedisRelayBus.d.ts +131 -0
  130. package/dist/types/util/RelayHub.d.ts +188 -0
  131. package/package.json +25 -11
@@ -28,7 +28,7 @@
28
28
  * ## Join announcement and roster discovery
29
29
  *
30
30
  * The channel has no history (a push event published while a socket was down is never replayed - see
31
- * `@rapidmx/react-shared`'s `pushClient.ts` doc comment for the identical guarantee on the mail push channel this
31
+ * `@rapidmx/web-client`'s `lib/mail/pushClient.ts` doc comment for the identical guarantee on the mail push channel this
32
32
  * one shares its transport with), so a newcomer's `hello` would only reach participants who happened to already be
33
33
  * subscribed *and* who joined before them - never the other way around. Every participant who learns of a
34
34
  * genuinely new peer (from any message, not just `hello`) therefore echoes its own `hello` right back, once, the
@@ -57,10 +57,37 @@
57
57
  * `presenter-release`) once it observes the winning claim. See `handlePresenterClaim()`.
58
58
  *
59
59
  * Sharing a screen is `setLocalTrack("video", screenTrack)`; stopping is `setLocalTrack("video", cameraTrack)`.
60
+ *
61
+ * ## Three media paths, tried in order
62
+ *
63
+ * Each pair of participants reaches for the best path that works, and `MeshParticipant.transport` says which one it
64
+ * got:
65
+ *
66
+ * 1. **Direct peer-to-peer** (`"p2p"`). The `RTCPeerConnection` is given the STUN servers and the TURN server
67
+ * together, and ICE ranks a direct pair above a relayed one, so this is what a pair gets whenever it can.
68
+ * 2. **Relayed through the TURN server** (`"turn"`). The same connection, on the relay candidate ICE falls back to
69
+ * when the direct pairs fail - a participant behind a symmetric NAT or a firewall that blocks direct UDP. Which of
70
+ * the two ICE picked is read from the selected candidate pair once connected (`RTCPeerConnectionLike.connectionType()`).
71
+ * 3. **Proxied by this server over a WebSocket** (`"websocket"`, `apps/shared/relay/`). For a participant whose
72
+ * network lets nothing but ordinary HTTPS out, neither of the above ever connects. A pair whose connection has not
73
+ * come up within `connectTimeoutMs`, has failed, or has stayed `disconnected` for `disconnectedGraceMs` gives up
74
+ * on WebRTC and switches to it, telling the other side with a `relay-fallback` message so both switch even when
75
+ * only one of them noticed. Nothing is renegotiated - the pair simply stops using the `RTCPeerConnection` - and the
76
+ * relay's stream replaces the one WebRTC would have filled (a `remote-stream` event again). When the relay is
77
+ * unavailable (turned off by the operator, or a browser without WebCodecs) the pair is marked `"failed"` and stays
78
+ * in the roster, rather than the participant vanishing with no explanation as it used to.
79
+ *
80
+ * A pair never moves back up: once on the relay it stays there for the rest of the call.
60
81
  */
61
- import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type SignalingChannel } from "./types.js";
82
+ import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type RelayTransportLike, type SignalingChannel } from "./types.js";
62
83
  /** The lexicographically smaller uid is always the offerer for that pair - see this module's doc comment. */
63
84
  export declare function isOfferer(selfId: string, peerId: string): boolean;
85
+ /** How long a connection may stay in `connecting` before the pair gives up on WebRTC. ICE with an unreachable TURN
86
+ * server can take the better part of a minute to report `failed`; a person waiting on a call will not. It is still long
87
+ * enough for a slow TURN allocation and a few round trips of signaling. */
88
+ export declare const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
89
+ /** How long a connection may stay `disconnected` (ICE lost the path but may still recover) before giving up on it. */
90
+ export declare const DEFAULT_DISCONNECTED_GRACE_MS = 8000;
64
91
  type MediaKind = "audio" | "video";
65
92
  export interface MeshConnectionManagerOptions {
66
93
  /** The authenticated uid - what `from` must be on the wire (see `SignalMessage.from`). */
@@ -81,6 +108,13 @@ export interface MeshConnectionManagerOptions {
81
108
  /** Builds the (initially empty) stream one peer's incoming tracks are gathered into. Defaults to
82
109
  * `new MediaStream()`; a test supplies a fake. */
83
110
  createMediaStream?: () => MediaStream;
111
+ /** The WebSocket media relay, the last-resort path - see this module's doc comment. Leave unset when the
112
+ * operator has turned it off; without one a pair WebRTC cannot connect is marked `"failed"`. */
113
+ relay?: RelayTransportLike;
114
+ /** Defaults to `DEFAULT_CONNECT_TIMEOUT_MS`. */
115
+ connectTimeoutMs?: number;
116
+ /** Defaults to `DEFAULT_DISCONNECTED_GRACE_MS`. */
117
+ disconnectedGraceMs?: number;
84
118
  }
85
119
  export declare class MeshConnectionManager {
86
120
  private readonly options;
@@ -131,6 +165,22 @@ export declare class MeshConnectionManager {
131
165
  private updatePeer;
132
166
  private handleBye;
133
167
  private createPeer;
168
+ /** Reacts to the peer connection's state: notes which path a connected pair got, and gives up on WebRTC for a
169
+ * pair that failed or stays disconnected - see `fallBack()`. Ignores a pair that already left WebRTC. */
170
+ private handleConnectionState;
171
+ /** Records whether a connected pair is direct or relayed through TURN, from the candidate pair ICE selected. */
172
+ private detectTransport;
173
+ private setTransport;
174
+ private clearTimers;
175
+ /**
176
+ * Gives up on WebRTC for `peer` and moves the pair to the WebSocket relay (the third path - see this module's doc
177
+ * comment), or marks it `"failed"` when there is no relay to move to. `notify` tells the other side, which may not
178
+ * have noticed yet; a pair told by the other side does not tell it back.
179
+ */
180
+ private fallBack;
181
+ private handleRelayFallback;
182
+ /** The relay only encodes and uploads the local media while at least one pair is actually using it. */
183
+ private syncRelaySending;
134
184
  private initiateOffer;
135
185
  private handleOffer;
136
186
  private handleAnswer;
@@ -32,7 +32,7 @@
32
32
  * ## Join announcement and roster discovery
33
33
  *
34
34
  * The channel has no history (a push event published while a socket was down is never replayed - see
35
- * `@rapidmx/react-shared`'s `pushClient.ts` doc comment for the identical guarantee on the mail push channel this
35
+ * `@rapidmx/web-client`'s `lib/mail/pushClient.ts` doc comment for the identical guarantee on the mail push channel this
36
36
  * one shares its transport with), so a newcomer's `hello` would only reach participants who happened to already be
37
37
  * subscribed *and* who joined before them - never the other way around. Every participant who learns of a
38
38
  * genuinely new peer (from any message, not just `hello`) therefore echoes its own `hello` right back, once, the
@@ -61,6 +61,27 @@
61
61
  * `presenter-release`) once it observes the winning claim. See `handlePresenterClaim()`.
62
62
  *
63
63
  * Sharing a screen is `setLocalTrack("video", screenTrack)`; stopping is `setLocalTrack("video", cameraTrack)`.
64
+ *
65
+ * ## Three media paths, tried in order
66
+ *
67
+ * Each pair of participants reaches for the best path that works, and `MeshParticipant.transport` says which one it
68
+ * got:
69
+ *
70
+ * 1. **Direct peer-to-peer** (`"p2p"`). The `RTCPeerConnection` is given the STUN servers and the TURN server
71
+ * together, and ICE ranks a direct pair above a relayed one, so this is what a pair gets whenever it can.
72
+ * 2. **Relayed through the TURN server** (`"turn"`). The same connection, on the relay candidate ICE falls back to
73
+ * when the direct pairs fail - a participant behind a symmetric NAT or a firewall that blocks direct UDP. Which of
74
+ * the two ICE picked is read from the selected candidate pair once connected (`RTCPeerConnectionLike.connectionType()`).
75
+ * 3. **Proxied by this server over a WebSocket** (`"websocket"`, `apps/shared/relay/`). For a participant whose
76
+ * network lets nothing but ordinary HTTPS out, neither of the above ever connects. A pair whose connection has not
77
+ * come up within `connectTimeoutMs`, has failed, or has stayed `disconnected` for `disconnectedGraceMs` gives up
78
+ * on WebRTC and switches to it, telling the other side with a `relay-fallback` message so both switch even when
79
+ * only one of them noticed. Nothing is renegotiated - the pair simply stops using the `RTCPeerConnection` - and the
80
+ * relay's stream replaces the one WebRTC would have filled (a `remote-stream` event again). When the relay is
81
+ * unavailable (turned off by the operator, or a browser without WebCodecs) the pair is marked `"failed"` and stays
82
+ * in the roster, rather than the participant vanishing with no explanation as it used to.
83
+ *
84
+ * A pair never moves back up: once on the relay it stays there for the rest of the call.
64
85
  */
65
86
  import { REACTION_EMOJIS, } from "./types.js";
66
87
  /** The lexicographically smaller uid is always the offerer for that pair - see this module's doc comment. */
@@ -71,6 +92,12 @@ export function isOfferer(selfId, peerId) {
71
92
  * misbehaving participant could make this hold for peers that never materialize. */
72
93
  const MAX_ORPHAN_PEERS = 32;
73
94
  const MAX_ORPHAN_CANDIDATES = 64;
95
+ /** How long a connection may stay in `connecting` before the pair gives up on WebRTC. ICE with an unreachable TURN
96
+ * server can take the better part of a minute to report `failed`; a person waiting on a call will not. It is still long
97
+ * enough for a slow TURN allocation and a few round trips of signaling. */
98
+ export const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
99
+ /** How long a connection may stay `disconnected` (ICE lost the path but may still recover) before giving up on it. */
100
+ export const DEFAULT_DISCONNECTED_GRACE_MS = 8000;
74
101
  export class MeshConnectionManager {
75
102
  constructor(options) {
76
103
  this.options = options;
@@ -87,6 +114,8 @@ export class MeshConnectionManager {
87
114
  handRaised: false,
88
115
  ...options.localState,
89
116
  };
117
+ options.relay?.setLocalTrack("audio", this.localTracks.audio);
118
+ options.relay?.setLocalTrack("video", this.localTracks.video);
90
119
  }
91
120
  get participants() {
92
121
  return [...this.peers.values()].map((peer) => toParticipant(peer));
@@ -122,11 +151,13 @@ export class MeshConnectionManager {
122
151
  this.unsubscribe?.();
123
152
  this.unsubscribe = undefined;
124
153
  for (const peer of this.peers.values()) {
154
+ this.clearTimers(peer);
125
155
  peer.pc.close();
126
156
  }
127
157
  this.peers.clear();
128
158
  this.orphanCandidates.clear();
129
159
  this.listeners.clear();
160
+ this.options.relay?.close();
130
161
  }
131
162
  /** Sets the track sent as `kind` (`null` to send nothing) on every connection, now and for every peer that
132
163
  * joins later - see this module's doc comment. */
@@ -135,6 +166,7 @@ export class MeshConnectionManager {
135
166
  for (const peer of this.peers.values()) {
136
167
  void peer.senders[kind]?.replaceTrack(track);
137
168
  }
169
+ this.options.relay?.setLocalTrack(kind, track);
138
170
  }
139
171
  /** Updates what this participant announces (`audioOn`/`videoOn`/`handRaised`) and tells everyone if anything
140
172
  * actually changed. Before `start()` it only records the state, which `hello` then carries. */
@@ -231,6 +263,9 @@ export class MeshConnectionManager {
231
263
  case "reaction":
232
264
  this.handleReaction(message);
233
265
  return;
266
+ case "relay-fallback":
267
+ this.handleRelayFallback(message.from);
268
+ return;
234
269
  }
235
270
  }
236
271
  handleHello(message) {
@@ -288,8 +323,13 @@ export class MeshConnectionManager {
288
323
  if (!peer) {
289
324
  return;
290
325
  }
326
+ this.clearTimers(peer);
291
327
  peer.pc.close();
292
328
  this.peers.delete(uid);
329
+ if (peer.transport === "websocket") {
330
+ this.options.relay?.stopReceivingFrom(uid);
331
+ this.syncRelaySending();
332
+ }
293
333
  this.emit({ type: "participant-left", uid });
294
334
  if (this.currentPresenterUid === uid) {
295
335
  this.currentPresenterUid = undefined;
@@ -312,6 +352,9 @@ export class MeshConnectionManager {
312
352
  remoteStream: (this.options.createMediaStream ?? (() => new MediaStream()))(),
313
353
  remoteDescriptionSet: false,
314
354
  pendingCandidates: this.orphanCandidates.get(uid) ?? [],
355
+ transport: "connecting",
356
+ connectTimer: undefined,
357
+ disconnectTimer: undefined,
315
358
  };
316
359
  this.orphanCandidates.delete(uid);
317
360
  this.peers.set(uid, peer);
@@ -326,13 +369,101 @@ export class MeshConnectionManager {
326
369
  }
327
370
  this.emit({ type: "remote-stream", uid, stream: peer.remoteStream });
328
371
  };
329
- pc.onconnectionstatechange = () => {
330
- if (pc.connectionState === "failed" || pc.connectionState === "closed") {
331
- this.handleBye(uid);
332
- }
333
- };
372
+ pc.onconnectionstatechange = () => this.handleConnectionState(peer);
373
+ // Cleared the moment the connection comes up, so this only ever fires for a pair still waiting on WebRTC.
374
+ peer.connectTimer = setTimeout(() => this.fallBack(peer), this.options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS);
334
375
  return peer;
335
376
  }
377
+ /** Reacts to the peer connection's state: notes which path a connected pair got, and gives up on WebRTC for a
378
+ * pair that failed or stays disconnected - see `fallBack()`. Ignores a pair that already left WebRTC. */
379
+ handleConnectionState(peer) {
380
+ if (peer.transport === "websocket" || this.peers.get(peer.uid) !== peer) {
381
+ return;
382
+ }
383
+ switch (peer.pc.connectionState) {
384
+ case "connected":
385
+ this.clearTimers(peer);
386
+ void this.detectTransport(peer);
387
+ return;
388
+ case "disconnected":
389
+ // ICE often recovers from this on its own, so it is only given up on after a grace period.
390
+ peer.disconnectTimer ?? (peer.disconnectTimer = setTimeout(() => this.fallBack(peer), this.options.disconnectedGraceMs ?? DEFAULT_DISCONNECTED_GRACE_MS));
391
+ return;
392
+ case "failed":
393
+ this.fallBack(peer);
394
+ return;
395
+ case "closed":
396
+ this.handleBye(peer.uid);
397
+ return;
398
+ }
399
+ }
400
+ /** Records whether a connected pair is direct or relayed through TURN, from the candidate pair ICE selected. */
401
+ async detectTransport(peer) {
402
+ let type;
403
+ try {
404
+ type = await peer.pc.connectionType();
405
+ }
406
+ catch {
407
+ type = "unknown";
408
+ }
409
+ // The pair may have moved on (fallen back, left, or dropped again) while the stats were being read.
410
+ if (peer.transport !== "websocket" && this.peers.get(peer.uid) === peer && peer.pc.connectionState === "connected") {
411
+ this.setTransport(peer, type === "turn" ? "turn" : "p2p");
412
+ }
413
+ }
414
+ setTransport(peer, transport) {
415
+ if (peer.transport !== transport) {
416
+ peer.transport = transport;
417
+ this.emit({ type: "participant-updated", participant: toParticipant(peer) });
418
+ }
419
+ }
420
+ clearTimers(peer) {
421
+ clearTimeout(peer.connectTimer);
422
+ clearTimeout(peer.disconnectTimer);
423
+ peer.connectTimer = undefined;
424
+ peer.disconnectTimer = undefined;
425
+ }
426
+ /**
427
+ * Gives up on WebRTC for `peer` and moves the pair to the WebSocket relay (the third path - see this module's doc
428
+ * comment), or marks it `"failed"` when there is no relay to move to. `notify` tells the other side, which may not
429
+ * have noticed yet; a pair told by the other side does not tell it back.
430
+ */
431
+ fallBack(peer, notify = true) {
432
+ if (peer.transport === "websocket" || this.peers.get(peer.uid) !== peer) {
433
+ return;
434
+ }
435
+ this.clearTimers(peer);
436
+ const relay = this.options.relay;
437
+ if (!relay?.supported) {
438
+ this.setTransport(peer, "failed");
439
+ return;
440
+ }
441
+ // Stop listening to the dead connection before closing it, so closing it cannot be mistaken for the peer leaving.
442
+ peer.pc.onconnectionstatechange = null;
443
+ peer.pc.onicecandidate = null;
444
+ peer.pc.ontrack = null;
445
+ peer.pc.close();
446
+ peer.transport = "websocket";
447
+ relay.receiveFrom(peer.uid, (stream) => {
448
+ peer.remoteStream = stream;
449
+ this.emit({ type: "remote-stream", uid: peer.uid, stream });
450
+ });
451
+ this.syncRelaySending();
452
+ this.emit({ type: "participant-updated", participant: toParticipant(peer) });
453
+ if (notify) {
454
+ this.send({ kind: "relay-fallback", to: peer.uid });
455
+ }
456
+ }
457
+ handleRelayFallback(from) {
458
+ const peer = this.peers.get(from);
459
+ if (peer) {
460
+ this.fallBack(peer, false);
461
+ }
462
+ }
463
+ /** The relay only encodes and uploads the local media while at least one pair is actually using it. */
464
+ syncRelaySending() {
465
+ this.options.relay?.setSending([...this.peers.values()].some((peer) => peer.transport === "websocket"));
466
+ }
336
467
  async initiateOffer(peer) {
337
468
  const offer = await peer.pc.createOffer();
338
469
  await peer.pc.setLocalDescription(offer);
@@ -438,7 +569,7 @@ export class MeshConnectionManager {
438
569
  }
439
570
  }
440
571
  function toParticipant(peer) {
441
- return { uid: peer.uid, name: peer.name, audioOn: peer.audioOn, videoOn: peer.videoOn, handRaised: peer.handRaised };
572
+ return { uid: peer.uid, name: peer.name, audioOn: peer.audioOn, videoOn: peer.videoOn, handRaised: peer.handRaised, transport: peer.transport };
442
573
  }
443
574
  function sameState(a, b) {
444
575
  return a.audioOn === b.audioOn && a.videoOn === b.videoOn && a.handRaised === b.handRaised;
@@ -4,4 +4,11 @@
4
4
  * module scope, so importing this file is safe during this plugin's page SSR, where `RTCPeerConnection` does not
5
5
  * exist at all - it would only throw if actually *called* outside a browser. */
6
6
  import type { RTCPeerConnectionFactory } from "./types.js";
7
+ /**
8
+ * Whether the pair ICE settled on goes through a TURN relay. The selected pair is found the standard way (the
9
+ * `transport` report's `selectedCandidatePairId`) and, for a browser that does not fill that in, as the succeeded and
10
+ * nominated `candidate-pair`. A relay on *either* end means the media passes through a TURN server, so it counts as
11
+ * `"turn"`; `"unknown"` when no pair is reported at all.
12
+ */
13
+ export declare function selectedConnectionType(pc: Pick<RTCPeerConnection, "getStats">): Promise<"p2p" | "turn" | "unknown">;
7
14
  export declare const createBrowserPeerConnection: RTCPeerConnectionFactory;
@@ -1,3 +1,22 @@
1
+ /**
2
+ * Whether the pair ICE settled on goes through a TURN relay. The selected pair is found the standard way (the
3
+ * `transport` report's `selectedCandidatePairId`) and, for a browser that does not fill that in, as the succeeded and
4
+ * nominated `candidate-pair`. A relay on *either* end means the media passes through a TURN server, so it counts as
5
+ * `"turn"`; `"unknown"` when no pair is reported at all.
6
+ */
7
+ export async function selectedConnectionType(pc) {
8
+ const reports = new Map();
9
+ (await pc.getStats()).forEach((report) => reports.set(String(report.id), report));
10
+ const transport = [...reports.values()].find((report) => report.type === "transport" && report.selectedCandidatePairId);
11
+ const pair = (transport ? reports.get(String(transport.selectedCandidatePairId)) : undefined) ??
12
+ [...reports.values()].find((report) => report.type === "candidate-pair" && (report.selected || (report.nominated && report.state === "succeeded")));
13
+ if (!pair) {
14
+ return "unknown";
15
+ }
16
+ const local = reports.get(String(pair.localCandidateId));
17
+ const remote = reports.get(String(pair.remoteCandidateId));
18
+ return local?.candidateType === "relay" || remote?.candidateType === "relay" ? "turn" : "p2p";
19
+ }
1
20
  export const createBrowserPeerConnection = (config) => {
2
21
  const pc = new RTCPeerConnection(config);
3
22
  const like = {
@@ -22,6 +41,7 @@ export const createBrowserPeerConnection = (config) => {
22
41
  setRemoteDescription: (description) => pc.setRemoteDescription(description),
23
42
  addIceCandidate: (candidate) => pc.addIceCandidate(candidate),
24
43
  close: () => pc.close(),
44
+ connectionType: () => selectedConnectionType(pc),
25
45
  onicecandidate: null,
26
46
  ontrack: null,
27
47
  onconnectionstatechange: null,
@@ -28,7 +28,7 @@ export declare const REACTION_EMOJIS: readonly ["👍", "👏", "❤️", "🎉"
28
28
  */
29
29
  export interface SignalMessage {
30
30
  type: "video-meeting-signal";
31
- kind: "hello" | "bye" | "offer" | "answer" | "ice-candidate" | "presenter-claim" | "presenter-release" | "state" | "reaction";
31
+ kind: "hello" | "bye" | "offer" | "answer" | "ice-candidate" | "presenter-claim" | "presenter-release" | "state" | "reaction" | "relay-fallback";
32
32
  /** The sender's authenticated uid - the real caller's own uid when `join()` returned `authenticated: true`, else
33
33
  * the `guest:<random>` uid `BaseVideoMeetingRoute.join()` minted. The server refuses a published message whose
34
34
  * `from` isn't the authenticated caller's uid (it stops one participant speaking as another), so this must be
@@ -39,7 +39,7 @@ export interface SignalMessage {
39
39
  * ignored, so it can't name someone else's tab. Absent from a sender that predates it (the participant is then
40
40
  * identified by `from`). */
41
41
  peer?: string;
42
- /** Set only on a point-to-point message (`offer`/`answer`/`ice-candidate`). */
42
+ /** Set only on a point-to-point message (`offer`/`answer`/`ice-candidate`/`relay-fallback`). */
43
43
  to?: string;
44
44
  /** `hello` only - the display name the sender chose in the lobby. */
45
45
  name?: string;
@@ -99,6 +99,31 @@ export interface RTCPeerConnectionLike {
99
99
  }) => void) | null;
100
100
  onconnectionstatechange: (() => void) | null;
101
101
  connectionState: string;
102
+ /** Which path the connected pair is using - read from the selected candidate pair once `connectionState` is
103
+ * `connected`. `"unknown"` when the browser does not report one (treated as direct by the caller). */
104
+ connectionType(): Promise<"p2p" | "turn" | "unknown">;
105
+ }
106
+ /** Which of the three media paths a participant's audio and video currently take, in the order they are tried:
107
+ *
108
+ * - `"p2p"`: a direct peer-to-peer WebRTC connection, the preferred path.
109
+ * - `"turn"`: still WebRTC, but relayed through the TURN server, for a participant a direct path cannot reach.
110
+ * - `"websocket"`: not WebRTC at all - encoded frames proxied by this server over a WebSocket (see
111
+ * `apps/shared/relay/`), the last resort when neither of the above connects.
112
+ * - `"connecting"`: no path established yet. `"failed"`: every path this browser can try has failed. */
113
+ export type MediaTransport = "connecting" | "p2p" | "turn" | "websocket" | "failed";
114
+ /** What `MeshConnectionManager` needs from the WebSocket media relay (`apps/shared/relay/RelayTransport.ts` implements
115
+ * it) - declared here so the mesh, and its tests, depend on the shape alone. */
116
+ export interface RelayTransportLike {
117
+ /** `false` when this browser cannot do it (no WebSocket, no WebCodecs) - the mesh then never falls back to it. */
118
+ readonly supported: boolean;
119
+ /** Starts receiving `peerId`'s media. `onStream` is given the `MediaStream` that plays it. */
120
+ receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void;
121
+ stopReceivingFrom(peerId: string): void;
122
+ /** Starts or stops publishing the local tracks to the relay. */
123
+ setSending(active: boolean): void;
124
+ /** Same as `MeshConnectionManager.setLocalTrack()` - `null` sends nothing for that kind. */
125
+ setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
126
+ close(): void;
102
127
  }
103
128
  export type RTCPeerConnectionFactory = (config: {
104
129
  iceServers: RTCIceServer[];
@@ -108,6 +133,8 @@ export type RTCPeerConnectionFactory = (config: {
108
133
  export interface MeshParticipant extends ParticipantState {
109
134
  uid: string;
110
135
  name: string;
136
+ /** How this participant's media currently reaches the local one - see `MediaTransport`. */
137
+ transport: MediaTransport;
111
138
  }
112
139
  export type MeshEvent = {
113
140
  type: "participant-joined";
package/dist/lib/index.js CHANGED
@@ -19,6 +19,8 @@ export * from "./models/types.js";
19
19
  export * from "./util/BookingIntegrationUtils.js";
20
20
  export * from "./util/IceServerUtils.js";
21
21
  export * from "./util/PublicUrlUtils.js";
22
+ export * from "./util/RedisRelayBus.js";
23
+ export * from "./util/RelayHub.js";
22
24
  export * from "./util/RouteAccessUtils.js";
23
25
  export * from "./util/TokenUtils.js";
24
26
  export * from "./routes/BaseVideoMeetingRoute.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,mEAAmE;AACnE,mCAAmC;AACnC,+EAA+E;AAC/E;;;;;;;;;;;;GAYG;AACH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,mCAAmC,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,mEAAmE;AACnE,mCAAmC;AACnC,+EAA+E;AAC/E;;;;;;;;;;;;GAYG;AACH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,mCAAmC,CAAC"}