@rapidmx/meet-plugin 0.6.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/apps/meet/[token].tsx +11 -10
- package/apps/meet/_CallControls.tsx +28 -2
- package/apps/meet/_CallView.tsx +26 -2
- package/apps/meet/_EffectsPanel.tsx +134 -0
- package/apps/meet/_MeetChrome.tsx +1 -1
- package/apps/meet/_MeetLobby.tsx +36 -4
- package/apps/meet/_ParticipantTile.tsx +73 -10
- package/apps/meet/_icons.tsx +4 -0
- package/apps/meet/_layout.tsx +2 -2
- package/apps/meet/_meetApi.ts +7 -1
- package/apps/meet/index.tsx +2 -2
- package/apps/settings-video-conferencing/_PersonalRoomCard.tsx +4 -4
- package/apps/settings-video-conferencing/_layout.tsx +2 -2
- package/apps/settings-video-conferencing/index.tsx +3 -3
- package/apps/shared/media/filters/VideoFilterProcessor.ts +408 -0
- package/apps/shared/media/filters/accessories.ts +262 -0
- package/apps/shared/media/filters/backgroundImage.ts +69 -0
- package/apps/shared/media/filters/filterTypes.ts +89 -0
- package/apps/shared/media/filters/mlModels.ts +174 -0
- package/apps/shared/media/filters/pixelEffects.ts +61 -0
- package/apps/shared/media/mediaPreferences.ts +105 -0
- package/apps/shared/media/useLocalMedia.ts +198 -14
- package/apps/shared/push/GuestSignalingClient.ts +4 -4
- package/apps/shared/relay/AudioPlayer.ts +111 -0
- package/apps/shared/relay/AudioSender.ts +194 -0
- package/apps/shared/relay/RelayClient.ts +262 -0
- package/apps/shared/relay/RelayReceiver.ts +72 -0
- package/apps/shared/relay/RelaySender.ts +74 -0
- package/apps/shared/relay/RelayTransport.ts +188 -0
- package/apps/shared/relay/VideoPlayer.ts +110 -0
- package/apps/shared/relay/VideoSender.ts +216 -0
- package/apps/shared/relay/audioResume.ts +55 -0
- package/apps/shared/relay/frames.ts +275 -0
- package/apps/shared/relay/relayEnv.ts +230 -0
- package/apps/shared/relay/senderCommon.ts +30 -0
- package/apps/shared/webrtc/MeshConnectionManager.ts +159 -7
- package/apps/shared/webrtc/realPeerConnection.ts +22 -0
- package/apps/shared/webrtc/types.ts +32 -2
- package/dist/apps/meet/[token].d.ts +3 -3
- package/dist/apps/meet/[token].js +10 -10
- package/dist/apps/meet/_CallControls.d.ts +2 -1
- package/dist/apps/meet/_CallControls.js +6 -3
- package/dist/apps/meet/_CallView.d.ts +4 -1
- package/dist/apps/meet/_CallView.js +20 -3
- package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
- package/dist/apps/meet/_EffectsPanel.js +60 -0
- package/dist/apps/meet/_MeetChrome.d.ts +1 -1
- package/dist/apps/meet/_MeetLobby.d.ts +8 -0
- package/dist/apps/meet/_MeetLobby.js +15 -4
- package/dist/apps/meet/_ParticipantTile.d.ts +17 -2
- package/dist/apps/meet/_ParticipantTile.js +27 -4
- package/dist/apps/meet/_icons.d.ts +1 -0
- package/dist/apps/meet/_icons.js +1 -0
- package/dist/apps/meet/_layout.d.ts +1 -1
- package/dist/apps/meet/_layout.js +1 -1
- package/dist/apps/meet/_meetApi.d.ts +6 -0
- package/dist/apps/meet/_meetApi.js +1 -1
- package/dist/apps/meet/index.js +2 -2
- package/dist/apps/settings-video-conferencing/_PersonalRoomCard.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/_PersonalRoomCard.js +4 -4
- package/dist/apps/settings-video-conferencing/_layout.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/_layout.js +1 -1
- package/dist/apps/settings-video-conferencing/index.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/index.js +3 -3
- package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +78 -0
- package/dist/apps/shared/media/filters/VideoFilterProcessor.js +329 -0
- package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
- package/dist/apps/shared/media/filters/accessories.js +211 -0
- package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
- package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
- package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
- package/dist/apps/shared/media/filters/filterTypes.js +63 -0
- package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
- package/dist/apps/shared/media/filters/mlModels.js +106 -0
- package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
- package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
- package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
- package/dist/apps/shared/media/mediaPreferences.js +93 -0
- package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
- package/dist/apps/shared/media/useLocalMedia.js +160 -14
- package/dist/apps/shared/push/GuestSignalingClient.d.ts +1 -1
- package/dist/apps/shared/push/GuestSignalingClient.js +3 -3
- package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
- package/dist/apps/shared/relay/AudioPlayer.js +103 -0
- package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
- package/dist/apps/shared/relay/AudioSender.js +172 -0
- package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
- package/dist/apps/shared/relay/RelayClient.js +236 -0
- package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
- package/dist/apps/shared/relay/RelayReceiver.js +59 -0
- package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
- package/dist/apps/shared/relay/RelaySender.js +54 -0
- package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
- package/dist/apps/shared/relay/RelayTransport.js +142 -0
- package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
- package/dist/apps/shared/relay/VideoPlayer.js +102 -0
- package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
- package/dist/apps/shared/relay/VideoSender.js +204 -0
- package/dist/apps/shared/relay/audioResume.d.ts +12 -0
- package/dist/apps/shared/relay/audioResume.js +45 -0
- package/dist/apps/shared/relay/frames.d.ts +101 -0
- package/dist/apps/shared/relay/frames.js +221 -0
- package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
- package/dist/apps/shared/relay/relayEnv.js +69 -0
- package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
- package/dist/apps/shared/relay/senderCommon.js +10 -0
- package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +52 -2
- package/dist/apps/shared/webrtc/MeshConnectionManager.js +138 -7
- package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
- package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
- package/dist/apps/shared/webrtc/types.d.ts +29 -2
- package/dist/lib/index.js +2 -0
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
- package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
- package/dist/lib/util/EffectsUtils.js +21 -0
- package/dist/lib/util/EffectsUtils.js.map +1 -0
- package/dist/lib/util/IceServerUtils.js +25 -4
- package/dist/lib/util/IceServerUtils.js.map +1 -1
- package/dist/lib/util/RedisRelayBus.js +241 -0
- package/dist/lib/util/RedisRelayBus.js.map +1 -0
- package/dist/lib/util/RelayHub.js +405 -0
- package/dist/lib/util/RelayHub.js.map +1 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
- package/dist/types/util/EffectsUtils.d.ts +10 -0
- package/dist/types/util/IceServerUtils.d.ts +12 -1
- package/dist/types/util/RedisRelayBus.d.ts +131 -0
- package/dist/types/util/RelayHub.d.ts +188 -0
- package/package.json +25 -11
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
* ## Join announcement and roster discovery
|
|
29
29
|
*
|
|
30
30
|
* The channel has no history (a push event published while a socket was down is never replayed - see
|
|
31
|
-
* `@rapidmx/
|
|
31
|
+
* `@rapidmx/web-client`'s `lib/mail/pushClient.ts` doc comment for the identical guarantee on the mail push channel this
|
|
32
32
|
* one shares its transport with), so a newcomer's `hello` would only reach participants who happened to already be
|
|
33
33
|
* subscribed *and* who joined before them - never the other way around. Every participant who learns of a
|
|
34
34
|
* genuinely new peer (from any message, not just `hello`) therefore echoes its own `hello` right back, once, the
|
|
@@ -57,10 +57,37 @@
|
|
|
57
57
|
* `presenter-release`) once it observes the winning claim. See `handlePresenterClaim()`.
|
|
58
58
|
*
|
|
59
59
|
* Sharing a screen is `setLocalTrack("video", screenTrack)`; stopping is `setLocalTrack("video", cameraTrack)`.
|
|
60
|
+
*
|
|
61
|
+
* ## Three media paths, tried in order
|
|
62
|
+
*
|
|
63
|
+
* Each pair of participants reaches for the best path that works, and `MeshParticipant.transport` says which one it
|
|
64
|
+
* got:
|
|
65
|
+
*
|
|
66
|
+
* 1. **Direct peer-to-peer** (`"p2p"`). The `RTCPeerConnection` is given the STUN servers and the TURN server
|
|
67
|
+
* together, and ICE ranks a direct pair above a relayed one, so this is what a pair gets whenever it can.
|
|
68
|
+
* 2. **Relayed through the TURN server** (`"turn"`). The same connection, on the relay candidate ICE falls back to
|
|
69
|
+
* when the direct pairs fail - a participant behind a symmetric NAT or a firewall that blocks direct UDP. Which of
|
|
70
|
+
* the two ICE picked is read from the selected candidate pair once connected (`RTCPeerConnectionLike.connectionType()`).
|
|
71
|
+
* 3. **Proxied by this server over a WebSocket** (`"websocket"`, `apps/shared/relay/`). For a participant whose
|
|
72
|
+
* network lets nothing but ordinary HTTPS out, neither of the above ever connects. A pair whose connection has not
|
|
73
|
+
* come up within `connectTimeoutMs`, has failed, or has stayed `disconnected` for `disconnectedGraceMs` gives up
|
|
74
|
+
* on WebRTC and switches to it, telling the other side with a `relay-fallback` message so both switch even when
|
|
75
|
+
* only one of them noticed. Nothing is renegotiated - the pair simply stops using the `RTCPeerConnection` - and the
|
|
76
|
+
* relay's stream replaces the one WebRTC would have filled (a `remote-stream` event again). When the relay is
|
|
77
|
+
* unavailable (turned off by the operator, or a browser without WebCodecs) the pair is marked `"failed"` and stays
|
|
78
|
+
* in the roster, rather than the participant vanishing with no explanation as it used to.
|
|
79
|
+
*
|
|
80
|
+
* A pair never moves back up: once on the relay it stays there for the rest of the call.
|
|
60
81
|
*/
|
|
61
|
-
import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type SignalingChannel } from "./types.js";
|
|
82
|
+
import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type RelayTransportLike, type SignalingChannel } from "./types.js";
|
|
62
83
|
/** The lexicographically smaller uid is always the offerer for that pair - see this module's doc comment. */
|
|
63
84
|
export declare function isOfferer(selfId: string, peerId: string): boolean;
|
|
85
|
+
/** How long a connection may stay in `connecting` before the pair gives up on WebRTC. ICE with an unreachable TURN
|
|
86
|
+
* server can take the better part of a minute to report `failed`; a person waiting on a call will not. It is still long
|
|
87
|
+
* enough for a slow TURN allocation and a few round trips of signaling. */
|
|
88
|
+
export declare const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
|
|
89
|
+
/** How long a connection may stay `disconnected` (ICE lost the path but may still recover) before giving up on it. */
|
|
90
|
+
export declare const DEFAULT_DISCONNECTED_GRACE_MS = 8000;
|
|
64
91
|
type MediaKind = "audio" | "video";
|
|
65
92
|
export interface MeshConnectionManagerOptions {
|
|
66
93
|
/** The authenticated uid - what `from` must be on the wire (see `SignalMessage.from`). */
|
|
@@ -81,6 +108,13 @@ export interface MeshConnectionManagerOptions {
|
|
|
81
108
|
/** Builds the (initially empty) stream one peer's incoming tracks are gathered into. Defaults to
|
|
82
109
|
* `new MediaStream()`; a test supplies a fake. */
|
|
83
110
|
createMediaStream?: () => MediaStream;
|
|
111
|
+
/** The WebSocket media relay, the last-resort path - see this module's doc comment. Leave unset when the
|
|
112
|
+
* operator has turned it off; without one a pair WebRTC cannot connect is marked `"failed"`. */
|
|
113
|
+
relay?: RelayTransportLike;
|
|
114
|
+
/** Defaults to `DEFAULT_CONNECT_TIMEOUT_MS`. */
|
|
115
|
+
connectTimeoutMs?: number;
|
|
116
|
+
/** Defaults to `DEFAULT_DISCONNECTED_GRACE_MS`. */
|
|
117
|
+
disconnectedGraceMs?: number;
|
|
84
118
|
}
|
|
85
119
|
export declare class MeshConnectionManager {
|
|
86
120
|
private readonly options;
|
|
@@ -131,6 +165,22 @@ export declare class MeshConnectionManager {
|
|
|
131
165
|
private updatePeer;
|
|
132
166
|
private handleBye;
|
|
133
167
|
private createPeer;
|
|
168
|
+
/** Reacts to the peer connection's state: notes which path a connected pair got, and gives up on WebRTC for a
|
|
169
|
+
* pair that failed or stays disconnected - see `fallBack()`. Ignores a pair that already left WebRTC. */
|
|
170
|
+
private handleConnectionState;
|
|
171
|
+
/** Records whether a connected pair is direct or relayed through TURN, from the candidate pair ICE selected. */
|
|
172
|
+
private detectTransport;
|
|
173
|
+
private setTransport;
|
|
174
|
+
private clearTimers;
|
|
175
|
+
/**
|
|
176
|
+
* Gives up on WebRTC for `peer` and moves the pair to the WebSocket relay (the third path - see this module's doc
|
|
177
|
+
* comment), or marks it `"failed"` when there is no relay to move to. `notify` tells the other side, which may not
|
|
178
|
+
* have noticed yet; a pair told by the other side does not tell it back.
|
|
179
|
+
*/
|
|
180
|
+
private fallBack;
|
|
181
|
+
private handleRelayFallback;
|
|
182
|
+
/** The relay only encodes and uploads the local media while at least one pair is actually using it. */
|
|
183
|
+
private syncRelaySending;
|
|
134
184
|
private initiateOffer;
|
|
135
185
|
private handleOffer;
|
|
136
186
|
private handleAnswer;
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
* ## Join announcement and roster discovery
|
|
33
33
|
*
|
|
34
34
|
* The channel has no history (a push event published while a socket was down is never replayed - see
|
|
35
|
-
* `@rapidmx/
|
|
35
|
+
* `@rapidmx/web-client`'s `lib/mail/pushClient.ts` doc comment for the identical guarantee on the mail push channel this
|
|
36
36
|
* one shares its transport with), so a newcomer's `hello` would only reach participants who happened to already be
|
|
37
37
|
* subscribed *and* who joined before them - never the other way around. Every participant who learns of a
|
|
38
38
|
* genuinely new peer (from any message, not just `hello`) therefore echoes its own `hello` right back, once, the
|
|
@@ -61,6 +61,27 @@
|
|
|
61
61
|
* `presenter-release`) once it observes the winning claim. See `handlePresenterClaim()`.
|
|
62
62
|
*
|
|
63
63
|
* Sharing a screen is `setLocalTrack("video", screenTrack)`; stopping is `setLocalTrack("video", cameraTrack)`.
|
|
64
|
+
*
|
|
65
|
+
* ## Three media paths, tried in order
|
|
66
|
+
*
|
|
67
|
+
* Each pair of participants reaches for the best path that works, and `MeshParticipant.transport` says which one it
|
|
68
|
+
* got:
|
|
69
|
+
*
|
|
70
|
+
* 1. **Direct peer-to-peer** (`"p2p"`). The `RTCPeerConnection` is given the STUN servers and the TURN server
|
|
71
|
+
* together, and ICE ranks a direct pair above a relayed one, so this is what a pair gets whenever it can.
|
|
72
|
+
* 2. **Relayed through the TURN server** (`"turn"`). The same connection, on the relay candidate ICE falls back to
|
|
73
|
+
* when the direct pairs fail - a participant behind a symmetric NAT or a firewall that blocks direct UDP. Which of
|
|
74
|
+
* the two ICE picked is read from the selected candidate pair once connected (`RTCPeerConnectionLike.connectionType()`).
|
|
75
|
+
* 3. **Proxied by this server over a WebSocket** (`"websocket"`, `apps/shared/relay/`). For a participant whose
|
|
76
|
+
* network lets nothing but ordinary HTTPS out, neither of the above ever connects. A pair whose connection has not
|
|
77
|
+
* come up within `connectTimeoutMs`, has failed, or has stayed `disconnected` for `disconnectedGraceMs` gives up
|
|
78
|
+
* on WebRTC and switches to it, telling the other side with a `relay-fallback` message so both switch even when
|
|
79
|
+
* only one of them noticed. Nothing is renegotiated - the pair simply stops using the `RTCPeerConnection` - and the
|
|
80
|
+
* relay's stream replaces the one WebRTC would have filled (a `remote-stream` event again). When the relay is
|
|
81
|
+
* unavailable (turned off by the operator, or a browser without WebCodecs) the pair is marked `"failed"` and stays
|
|
82
|
+
* in the roster, rather than the participant vanishing with no explanation as it used to.
|
|
83
|
+
*
|
|
84
|
+
* A pair never moves back up: once on the relay it stays there for the rest of the call.
|
|
64
85
|
*/
|
|
65
86
|
import { REACTION_EMOJIS, } from "./types.js";
|
|
66
87
|
/** The lexicographically smaller uid is always the offerer for that pair - see this module's doc comment. */
|
|
@@ -71,6 +92,12 @@ export function isOfferer(selfId, peerId) {
|
|
|
71
92
|
* misbehaving participant could make this hold for peers that never materialize. */
|
|
72
93
|
const MAX_ORPHAN_PEERS = 32;
|
|
73
94
|
const MAX_ORPHAN_CANDIDATES = 64;
|
|
95
|
+
/** How long a connection may stay in `connecting` before the pair gives up on WebRTC. ICE with an unreachable TURN
|
|
96
|
+
* server can take the better part of a minute to report `failed`; a person waiting on a call will not. It is still long
|
|
97
|
+
* enough for a slow TURN allocation and a few round trips of signaling. */
|
|
98
|
+
export const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
|
|
99
|
+
/** How long a connection may stay `disconnected` (ICE lost the path but may still recover) before giving up on it. */
|
|
100
|
+
export const DEFAULT_DISCONNECTED_GRACE_MS = 8000;
|
|
74
101
|
export class MeshConnectionManager {
|
|
75
102
|
constructor(options) {
|
|
76
103
|
this.options = options;
|
|
@@ -87,6 +114,8 @@ export class MeshConnectionManager {
|
|
|
87
114
|
handRaised: false,
|
|
88
115
|
...options.localState,
|
|
89
116
|
};
|
|
117
|
+
options.relay?.setLocalTrack("audio", this.localTracks.audio);
|
|
118
|
+
options.relay?.setLocalTrack("video", this.localTracks.video);
|
|
90
119
|
}
|
|
91
120
|
get participants() {
|
|
92
121
|
return [...this.peers.values()].map((peer) => toParticipant(peer));
|
|
@@ -122,11 +151,13 @@ export class MeshConnectionManager {
|
|
|
122
151
|
this.unsubscribe?.();
|
|
123
152
|
this.unsubscribe = undefined;
|
|
124
153
|
for (const peer of this.peers.values()) {
|
|
154
|
+
this.clearTimers(peer);
|
|
125
155
|
peer.pc.close();
|
|
126
156
|
}
|
|
127
157
|
this.peers.clear();
|
|
128
158
|
this.orphanCandidates.clear();
|
|
129
159
|
this.listeners.clear();
|
|
160
|
+
this.options.relay?.close();
|
|
130
161
|
}
|
|
131
162
|
/** Sets the track sent as `kind` (`null` to send nothing) on every connection, now and for every peer that
|
|
132
163
|
* joins later - see this module's doc comment. */
|
|
@@ -135,6 +166,7 @@ export class MeshConnectionManager {
|
|
|
135
166
|
for (const peer of this.peers.values()) {
|
|
136
167
|
void peer.senders[kind]?.replaceTrack(track);
|
|
137
168
|
}
|
|
169
|
+
this.options.relay?.setLocalTrack(kind, track);
|
|
138
170
|
}
|
|
139
171
|
/** Updates what this participant announces (`audioOn`/`videoOn`/`handRaised`) and tells everyone if anything
|
|
140
172
|
* actually changed. Before `start()` it only records the state, which `hello` then carries. */
|
|
@@ -231,6 +263,9 @@ export class MeshConnectionManager {
|
|
|
231
263
|
case "reaction":
|
|
232
264
|
this.handleReaction(message);
|
|
233
265
|
return;
|
|
266
|
+
case "relay-fallback":
|
|
267
|
+
this.handleRelayFallback(message.from);
|
|
268
|
+
return;
|
|
234
269
|
}
|
|
235
270
|
}
|
|
236
271
|
handleHello(message) {
|
|
@@ -288,8 +323,13 @@ export class MeshConnectionManager {
|
|
|
288
323
|
if (!peer) {
|
|
289
324
|
return;
|
|
290
325
|
}
|
|
326
|
+
this.clearTimers(peer);
|
|
291
327
|
peer.pc.close();
|
|
292
328
|
this.peers.delete(uid);
|
|
329
|
+
if (peer.transport === "websocket") {
|
|
330
|
+
this.options.relay?.stopReceivingFrom(uid);
|
|
331
|
+
this.syncRelaySending();
|
|
332
|
+
}
|
|
293
333
|
this.emit({ type: "participant-left", uid });
|
|
294
334
|
if (this.currentPresenterUid === uid) {
|
|
295
335
|
this.currentPresenterUid = undefined;
|
|
@@ -312,6 +352,9 @@ export class MeshConnectionManager {
|
|
|
312
352
|
remoteStream: (this.options.createMediaStream ?? (() => new MediaStream()))(),
|
|
313
353
|
remoteDescriptionSet: false,
|
|
314
354
|
pendingCandidates: this.orphanCandidates.get(uid) ?? [],
|
|
355
|
+
transport: "connecting",
|
|
356
|
+
connectTimer: undefined,
|
|
357
|
+
disconnectTimer: undefined,
|
|
315
358
|
};
|
|
316
359
|
this.orphanCandidates.delete(uid);
|
|
317
360
|
this.peers.set(uid, peer);
|
|
@@ -326,13 +369,101 @@ export class MeshConnectionManager {
|
|
|
326
369
|
}
|
|
327
370
|
this.emit({ type: "remote-stream", uid, stream: peer.remoteStream });
|
|
328
371
|
};
|
|
329
|
-
pc.onconnectionstatechange = () =>
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
}
|
|
333
|
-
};
|
|
372
|
+
pc.onconnectionstatechange = () => this.handleConnectionState(peer);
|
|
373
|
+
// Cleared the moment the connection comes up, so this only ever fires for a pair still waiting on WebRTC.
|
|
374
|
+
peer.connectTimer = setTimeout(() => this.fallBack(peer), this.options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS);
|
|
334
375
|
return peer;
|
|
335
376
|
}
|
|
377
|
+
/** Reacts to the peer connection's state: notes which path a connected pair got, and gives up on WebRTC for a
|
|
378
|
+
* pair that failed or stays disconnected - see `fallBack()`. Ignores a pair that already left WebRTC. */
|
|
379
|
+
handleConnectionState(peer) {
|
|
380
|
+
if (peer.transport === "websocket" || this.peers.get(peer.uid) !== peer) {
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
switch (peer.pc.connectionState) {
|
|
384
|
+
case "connected":
|
|
385
|
+
this.clearTimers(peer);
|
|
386
|
+
void this.detectTransport(peer);
|
|
387
|
+
return;
|
|
388
|
+
case "disconnected":
|
|
389
|
+
// ICE often recovers from this on its own, so it is only given up on after a grace period.
|
|
390
|
+
peer.disconnectTimer ?? (peer.disconnectTimer = setTimeout(() => this.fallBack(peer), this.options.disconnectedGraceMs ?? DEFAULT_DISCONNECTED_GRACE_MS));
|
|
391
|
+
return;
|
|
392
|
+
case "failed":
|
|
393
|
+
this.fallBack(peer);
|
|
394
|
+
return;
|
|
395
|
+
case "closed":
|
|
396
|
+
this.handleBye(peer.uid);
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
/** Records whether a connected pair is direct or relayed through TURN, from the candidate pair ICE selected. */
|
|
401
|
+
async detectTransport(peer) {
|
|
402
|
+
let type;
|
|
403
|
+
try {
|
|
404
|
+
type = await peer.pc.connectionType();
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
type = "unknown";
|
|
408
|
+
}
|
|
409
|
+
// The pair may have moved on (fallen back, left, or dropped again) while the stats were being read.
|
|
410
|
+
if (peer.transport !== "websocket" && this.peers.get(peer.uid) === peer && peer.pc.connectionState === "connected") {
|
|
411
|
+
this.setTransport(peer, type === "turn" ? "turn" : "p2p");
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
setTransport(peer, transport) {
|
|
415
|
+
if (peer.transport !== transport) {
|
|
416
|
+
peer.transport = transport;
|
|
417
|
+
this.emit({ type: "participant-updated", participant: toParticipant(peer) });
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
clearTimers(peer) {
|
|
421
|
+
clearTimeout(peer.connectTimer);
|
|
422
|
+
clearTimeout(peer.disconnectTimer);
|
|
423
|
+
peer.connectTimer = undefined;
|
|
424
|
+
peer.disconnectTimer = undefined;
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Gives up on WebRTC for `peer` and moves the pair to the WebSocket relay (the third path - see this module's doc
|
|
428
|
+
* comment), or marks it `"failed"` when there is no relay to move to. `notify` tells the other side, which may not
|
|
429
|
+
* have noticed yet; a pair told by the other side does not tell it back.
|
|
430
|
+
*/
|
|
431
|
+
fallBack(peer, notify = true) {
|
|
432
|
+
if (peer.transport === "websocket" || this.peers.get(peer.uid) !== peer) {
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
this.clearTimers(peer);
|
|
436
|
+
const relay = this.options.relay;
|
|
437
|
+
if (!relay?.supported) {
|
|
438
|
+
this.setTransport(peer, "failed");
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
// Stop listening to the dead connection before closing it, so closing it cannot be mistaken for the peer leaving.
|
|
442
|
+
peer.pc.onconnectionstatechange = null;
|
|
443
|
+
peer.pc.onicecandidate = null;
|
|
444
|
+
peer.pc.ontrack = null;
|
|
445
|
+
peer.pc.close();
|
|
446
|
+
peer.transport = "websocket";
|
|
447
|
+
relay.receiveFrom(peer.uid, (stream) => {
|
|
448
|
+
peer.remoteStream = stream;
|
|
449
|
+
this.emit({ type: "remote-stream", uid: peer.uid, stream });
|
|
450
|
+
});
|
|
451
|
+
this.syncRelaySending();
|
|
452
|
+
this.emit({ type: "participant-updated", participant: toParticipant(peer) });
|
|
453
|
+
if (notify) {
|
|
454
|
+
this.send({ kind: "relay-fallback", to: peer.uid });
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
handleRelayFallback(from) {
|
|
458
|
+
const peer = this.peers.get(from);
|
|
459
|
+
if (peer) {
|
|
460
|
+
this.fallBack(peer, false);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
/** The relay only encodes and uploads the local media while at least one pair is actually using it. */
|
|
464
|
+
syncRelaySending() {
|
|
465
|
+
this.options.relay?.setSending([...this.peers.values()].some((peer) => peer.transport === "websocket"));
|
|
466
|
+
}
|
|
336
467
|
async initiateOffer(peer) {
|
|
337
468
|
const offer = await peer.pc.createOffer();
|
|
338
469
|
await peer.pc.setLocalDescription(offer);
|
|
@@ -438,7 +569,7 @@ export class MeshConnectionManager {
|
|
|
438
569
|
}
|
|
439
570
|
}
|
|
440
571
|
function toParticipant(peer) {
|
|
441
|
-
return { uid: peer.uid, name: peer.name, audioOn: peer.audioOn, videoOn: peer.videoOn, handRaised: peer.handRaised };
|
|
572
|
+
return { uid: peer.uid, name: peer.name, audioOn: peer.audioOn, videoOn: peer.videoOn, handRaised: peer.handRaised, transport: peer.transport };
|
|
442
573
|
}
|
|
443
574
|
function sameState(a, b) {
|
|
444
575
|
return a.audioOn === b.audioOn && a.videoOn === b.videoOn && a.handRaised === b.handRaised;
|
|
@@ -4,4 +4,11 @@
|
|
|
4
4
|
* module scope, so importing this file is safe during this plugin's page SSR, where `RTCPeerConnection` does not
|
|
5
5
|
* exist at all - it would only throw if actually *called* outside a browser. */
|
|
6
6
|
import type { RTCPeerConnectionFactory } from "./types.js";
|
|
7
|
+
/**
|
|
8
|
+
* Whether the pair ICE settled on goes through a TURN relay. The selected pair is found the standard way (the
|
|
9
|
+
* `transport` report's `selectedCandidatePairId`) and, for a browser that does not fill that in, as the succeeded and
|
|
10
|
+
* nominated `candidate-pair`. A relay on *either* end means the media passes through a TURN server, so it counts as
|
|
11
|
+
* `"turn"`; `"unknown"` when no pair is reported at all.
|
|
12
|
+
*/
|
|
13
|
+
export declare function selectedConnectionType(pc: Pick<RTCPeerConnection, "getStats">): Promise<"p2p" | "turn" | "unknown">;
|
|
7
14
|
export declare const createBrowserPeerConnection: RTCPeerConnectionFactory;
|
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether the pair ICE settled on goes through a TURN relay. The selected pair is found the standard way (the
|
|
3
|
+
* `transport` report's `selectedCandidatePairId`) and, for a browser that does not fill that in, as the succeeded and
|
|
4
|
+
* nominated `candidate-pair`. A relay on *either* end means the media passes through a TURN server, so it counts as
|
|
5
|
+
* `"turn"`; `"unknown"` when no pair is reported at all.
|
|
6
|
+
*/
|
|
7
|
+
export async function selectedConnectionType(pc) {
|
|
8
|
+
const reports = new Map();
|
|
9
|
+
(await pc.getStats()).forEach((report) => reports.set(String(report.id), report));
|
|
10
|
+
const transport = [...reports.values()].find((report) => report.type === "transport" && report.selectedCandidatePairId);
|
|
11
|
+
const pair = (transport ? reports.get(String(transport.selectedCandidatePairId)) : undefined) ??
|
|
12
|
+
[...reports.values()].find((report) => report.type === "candidate-pair" && (report.selected || (report.nominated && report.state === "succeeded")));
|
|
13
|
+
if (!pair) {
|
|
14
|
+
return "unknown";
|
|
15
|
+
}
|
|
16
|
+
const local = reports.get(String(pair.localCandidateId));
|
|
17
|
+
const remote = reports.get(String(pair.remoteCandidateId));
|
|
18
|
+
return local?.candidateType === "relay" || remote?.candidateType === "relay" ? "turn" : "p2p";
|
|
19
|
+
}
|
|
1
20
|
export const createBrowserPeerConnection = (config) => {
|
|
2
21
|
const pc = new RTCPeerConnection(config);
|
|
3
22
|
const like = {
|
|
@@ -22,6 +41,7 @@ export const createBrowserPeerConnection = (config) => {
|
|
|
22
41
|
setRemoteDescription: (description) => pc.setRemoteDescription(description),
|
|
23
42
|
addIceCandidate: (candidate) => pc.addIceCandidate(candidate),
|
|
24
43
|
close: () => pc.close(),
|
|
44
|
+
connectionType: () => selectedConnectionType(pc),
|
|
25
45
|
onicecandidate: null,
|
|
26
46
|
ontrack: null,
|
|
27
47
|
onconnectionstatechange: null,
|
|
@@ -28,7 +28,7 @@ export declare const REACTION_EMOJIS: readonly ["👍", "👏", "❤️", "🎉"
|
|
|
28
28
|
*/
|
|
29
29
|
export interface SignalMessage {
|
|
30
30
|
type: "video-meeting-signal";
|
|
31
|
-
kind: "hello" | "bye" | "offer" | "answer" | "ice-candidate" | "presenter-claim" | "presenter-release" | "state" | "reaction";
|
|
31
|
+
kind: "hello" | "bye" | "offer" | "answer" | "ice-candidate" | "presenter-claim" | "presenter-release" | "state" | "reaction" | "relay-fallback";
|
|
32
32
|
/** The sender's authenticated uid - the real caller's own uid when `join()` returned `authenticated: true`, else
|
|
33
33
|
* the `guest:<random>` uid `BaseVideoMeetingRoute.join()` minted. The server refuses a published message whose
|
|
34
34
|
* `from` isn't the authenticated caller's uid (it stops one participant speaking as another), so this must be
|
|
@@ -39,7 +39,7 @@ export interface SignalMessage {
|
|
|
39
39
|
* ignored, so it can't name someone else's tab. Absent from a sender that predates it (the participant is then
|
|
40
40
|
* identified by `from`). */
|
|
41
41
|
peer?: string;
|
|
42
|
-
/** Set only on a point-to-point message (`offer`/`answer`/`ice-candidate`). */
|
|
42
|
+
/** Set only on a point-to-point message (`offer`/`answer`/`ice-candidate`/`relay-fallback`). */
|
|
43
43
|
to?: string;
|
|
44
44
|
/** `hello` only - the display name the sender chose in the lobby. */
|
|
45
45
|
name?: string;
|
|
@@ -99,6 +99,31 @@ export interface RTCPeerConnectionLike {
|
|
|
99
99
|
}) => void) | null;
|
|
100
100
|
onconnectionstatechange: (() => void) | null;
|
|
101
101
|
connectionState: string;
|
|
102
|
+
/** Which path the connected pair is using - read from the selected candidate pair once `connectionState` is
|
|
103
|
+
* `connected`. `"unknown"` when the browser does not report one (treated as direct by the caller). */
|
|
104
|
+
connectionType(): Promise<"p2p" | "turn" | "unknown">;
|
|
105
|
+
}
|
|
106
|
+
/** Which of the three media paths a participant's audio and video currently take, in the order they are tried:
|
|
107
|
+
*
|
|
108
|
+
* - `"p2p"`: a direct peer-to-peer WebRTC connection, the preferred path.
|
|
109
|
+
* - `"turn"`: still WebRTC, but relayed through the TURN server, for a participant a direct path cannot reach.
|
|
110
|
+
* - `"websocket"`: not WebRTC at all - encoded frames proxied by this server over a WebSocket (see
|
|
111
|
+
* `apps/shared/relay/`), the last resort when neither of the above connects.
|
|
112
|
+
* - `"connecting"`: no path established yet. `"failed"`: every path this browser can try has failed. */
|
|
113
|
+
export type MediaTransport = "connecting" | "p2p" | "turn" | "websocket" | "failed";
|
|
114
|
+
/** What `MeshConnectionManager` needs from the WebSocket media relay (`apps/shared/relay/RelayTransport.ts` implements
|
|
115
|
+
* it) - declared here so the mesh, and its tests, depend on the shape alone. */
|
|
116
|
+
export interface RelayTransportLike {
|
|
117
|
+
/** `false` when this browser cannot do it (no WebSocket, no WebCodecs) - the mesh then never falls back to it. */
|
|
118
|
+
readonly supported: boolean;
|
|
119
|
+
/** Starts receiving `peerId`'s media. `onStream` is given the `MediaStream` that plays it. */
|
|
120
|
+
receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void;
|
|
121
|
+
stopReceivingFrom(peerId: string): void;
|
|
122
|
+
/** Starts or stops publishing the local tracks to the relay. */
|
|
123
|
+
setSending(active: boolean): void;
|
|
124
|
+
/** Same as `MeshConnectionManager.setLocalTrack()` - `null` sends nothing for that kind. */
|
|
125
|
+
setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
|
|
126
|
+
close(): void;
|
|
102
127
|
}
|
|
103
128
|
export type RTCPeerConnectionFactory = (config: {
|
|
104
129
|
iceServers: RTCIceServer[];
|
|
@@ -108,6 +133,8 @@ export type RTCPeerConnectionFactory = (config: {
|
|
|
108
133
|
export interface MeshParticipant extends ParticipantState {
|
|
109
134
|
uid: string;
|
|
110
135
|
name: string;
|
|
136
|
+
/** How this participant's media currently reaches the local one - see `MediaTransport`. */
|
|
137
|
+
transport: MediaTransport;
|
|
111
138
|
}
|
|
112
139
|
export type MeshEvent = {
|
|
113
140
|
type: "participant-joined";
|
package/dist/lib/index.js
CHANGED
|
@@ -19,6 +19,8 @@ export * from "./models/types.js";
|
|
|
19
19
|
export * from "./util/BookingIntegrationUtils.js";
|
|
20
20
|
export * from "./util/IceServerUtils.js";
|
|
21
21
|
export * from "./util/PublicUrlUtils.js";
|
|
22
|
+
export * from "./util/RedisRelayBus.js";
|
|
23
|
+
export * from "./util/RelayHub.js";
|
|
22
24
|
export * from "./util/RouteAccessUtils.js";
|
|
23
25
|
export * from "./util/TokenUtils.js";
|
|
24
26
|
export * from "./routes/BaseVideoMeetingRoute.js";
|
package/dist/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,mEAAmE;AACnE,mCAAmC;AACnC,+EAA+E;AAC/E;;;;;;;;;;;;GAYG;AACH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,mCAAmC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,mEAAmE;AACnE,mCAAmC;AACnC,+EAA+E;AAC/E;;;;;;;;;;;;GAYG;AACH,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,mCAAmC,CAAC"}
|