@rapidmx/meet-plugin 0.5.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -2
- package/apps/meet/[token].tsx +3 -2
- package/apps/meet/_CallControls.tsx +28 -2
- package/apps/meet/_CallView.tsx +23 -2
- package/apps/meet/_EffectsPanel.tsx +134 -0
- package/apps/meet/_MeetLobby.tsx +22 -0
- package/apps/meet/_ParticipantTile.tsx +62 -9
- package/apps/meet/_icons.tsx +4 -0
- package/apps/meet/_meetApi.ts +6 -0
- package/apps/shared/media/filters/VideoFilterProcessor.ts +383 -0
- package/apps/shared/media/filters/accessories.ts +262 -0
- package/apps/shared/media/filters/backgroundImage.ts +69 -0
- package/apps/shared/media/filters/filterTypes.ts +89 -0
- package/apps/shared/media/filters/mlModels.ts +174 -0
- package/apps/shared/media/filters/pixelEffects.ts +61 -0
- package/apps/shared/media/mediaPreferences.ts +105 -0
- package/apps/shared/media/useLocalMedia.ts +198 -14
- package/apps/shared/relay/AudioPlayer.ts +111 -0
- package/apps/shared/relay/AudioSender.ts +194 -0
- package/apps/shared/relay/RelayClient.ts +262 -0
- package/apps/shared/relay/RelayReceiver.ts +72 -0
- package/apps/shared/relay/RelaySender.ts +74 -0
- package/apps/shared/relay/RelayTransport.ts +188 -0
- package/apps/shared/relay/VideoPlayer.ts +110 -0
- package/apps/shared/relay/VideoSender.ts +216 -0
- package/apps/shared/relay/audioResume.ts +55 -0
- package/apps/shared/relay/frames.ts +275 -0
- package/apps/shared/relay/relayEnv.ts +230 -0
- package/apps/shared/relay/senderCommon.ts +30 -0
- package/apps/shared/webrtc/MeshConnectionManager.ts +158 -6
- package/apps/shared/webrtc/realPeerConnection.ts +22 -0
- package/apps/shared/webrtc/types.ts +32 -2
- package/dist/apps/meet/[token].js +3 -3
- package/dist/apps/meet/_CallControls.d.ts +2 -1
- package/dist/apps/meet/_CallControls.js +6 -3
- package/dist/apps/meet/_CallView.d.ts +4 -1
- package/dist/apps/meet/_CallView.js +17 -3
- package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
- package/dist/apps/meet/_EffectsPanel.js +60 -0
- package/dist/apps/meet/_MeetLobby.d.ts +4 -0
- package/dist/apps/meet/_MeetLobby.js +8 -1
- package/dist/apps/meet/_ParticipantTile.d.ts +7 -1
- package/dist/apps/meet/_ParticipantTile.js +20 -3
- package/dist/apps/meet/_icons.d.ts +1 -0
- package/dist/apps/meet/_icons.js +1 -0
- package/dist/apps/meet/_meetApi.d.ts +6 -0
- package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +75 -0
- package/dist/apps/shared/media/filters/VideoFilterProcessor.js +304 -0
- package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
- package/dist/apps/shared/media/filters/accessories.js +211 -0
- package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
- package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
- package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
- package/dist/apps/shared/media/filters/filterTypes.js +63 -0
- package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
- package/dist/apps/shared/media/filters/mlModels.js +106 -0
- package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
- package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
- package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
- package/dist/apps/shared/media/mediaPreferences.js +93 -0
- package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
- package/dist/apps/shared/media/useLocalMedia.js +160 -14
- package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
- package/dist/apps/shared/relay/AudioPlayer.js +103 -0
- package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
- package/dist/apps/shared/relay/AudioSender.js +172 -0
- package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
- package/dist/apps/shared/relay/RelayClient.js +236 -0
- package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
- package/dist/apps/shared/relay/RelayReceiver.js +59 -0
- package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
- package/dist/apps/shared/relay/RelaySender.js +54 -0
- package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
- package/dist/apps/shared/relay/RelayTransport.js +142 -0
- package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
- package/dist/apps/shared/relay/VideoPlayer.js +102 -0
- package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
- package/dist/apps/shared/relay/VideoSender.js +204 -0
- package/dist/apps/shared/relay/audioResume.d.ts +12 -0
- package/dist/apps/shared/relay/audioResume.js +45 -0
- package/dist/apps/shared/relay/frames.d.ts +101 -0
- package/dist/apps/shared/relay/frames.js +221 -0
- package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
- package/dist/apps/shared/relay/relayEnv.js +69 -0
- package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
- package/dist/apps/shared/relay/senderCommon.js +10 -0
- package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +51 -1
- package/dist/apps/shared/webrtc/MeshConnectionManager.js +137 -6
- package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
- package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
- package/dist/apps/shared/webrtc/types.d.ts +29 -2
- package/dist/lib/index.js +2 -0
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
- package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
- package/dist/lib/util/EffectsUtils.js +21 -0
- package/dist/lib/util/EffectsUtils.js.map +1 -0
- package/dist/lib/util/IceServerUtils.js +25 -4
- package/dist/lib/util/IceServerUtils.js.map +1 -1
- package/dist/lib/util/RedisRelayBus.js +241 -0
- package/dist/lib/util/RedisRelayBus.js.map +1 -0
- package/dist/lib/util/RelayHub.js +405 -0
- package/dist/lib/util/RelayHub.js.map +1 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
- package/dist/types/util/EffectsUtils.d.ts +10 -0
- package/dist/types/util/IceServerUtils.d.ts +12 -1
- package/dist/types/util/RedisRelayBus.d.ts +131 -0
- package/dist/types/util/RelayHub.d.ts +188 -0
- package/package.json +24 -7
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* The WebSocket to the server's media relay - the transport under the last-resort tier of a call, used for a pair of
|
|
7
|
+
* participants whose WebRTC connection could not be made to work at all.
|
|
8
|
+
*
|
|
9
|
+
* ## Wire protocol (fixed by the server)
|
|
10
|
+
*
|
|
11
|
+
* - The first message is text JSON `{"op":"hello","v":1,"peer":"<peerId>"}`; the server answers `{"op":"ready","v":1}`,
|
|
12
|
+
* plus `"maxMessageBytes"` - the largest client-to-server message it accepts - on a current server (absent on an
|
|
13
|
+
* older one, which accepts `MAX_MESSAGE_BYTES`). Nothing else may be sent before `ready`.
|
|
14
|
+
* - `{"op":"want","peers":[...]}` REPLACES the set of senders this socket receives from (the server accepts at most
|
|
15
|
+
* `MAX_WANTED_PEERS`). It has to be re-sent after every reconnect, because the server's set dies with the socket.
|
|
16
|
+
* - A binary message from the client is an opaque payload of at most `maxMessageBytes`. The server forwards it to
|
|
17
|
+
* every socket that wants this sender as `[1 byte N][N bytes UTF-8 sender peer id][payload]`, stamping the sender
|
|
18
|
+
* itself so a peer cannot speak as another.
|
|
19
|
+
* - Binary messages may be dropped under backpressure and the socket may close at any moment, so this client
|
|
20
|
+
* reconnects with exponential backoff and treats every media send as best effort.
|
|
21
|
+
*
|
|
22
|
+
* Authentication is the `jwt` cookie already in place (see `GuestSignalingClient`), so this client sets no cookie.
|
|
23
|
+
*/
|
|
24
|
+
import { DEFAULT_MESSAGE_BYTES } from "./frames.js";
|
|
25
|
+
import type { RelaySocket } from "./relayEnv.js";
|
|
26
|
+
|
|
27
|
+
/** The largest binary message an older server accepts - it closes the socket on a larger one (uWS's default limit).
|
|
28
|
+
* Assumed until the server's `ready` announces a different `maxMessageBytes`. */
|
|
29
|
+
export const MAX_MESSAGE_BYTES = DEFAULT_MESSAGE_BYTES;
|
|
30
|
+
/** A `maxMessageBytes` below this is not believed (nothing useful fits), and one above
|
|
31
|
+
* `MAX_NEGOTIATED_MESSAGE_BYTES` is clamped to it. */
|
|
32
|
+
export const MIN_NEGOTIATED_MESSAGE_BYTES = 1024;
|
|
33
|
+
export const MAX_NEGOTIATED_MESSAGE_BYTES = 1024 * 1024;
|
|
34
|
+
/** The most senders the server lets one socket `want`. */
|
|
35
|
+
export const MAX_WANTED_PEERS = 32;
|
|
36
|
+
/** A media send is dropped, not queued, while more than this many bytes are still waiting to leave - the
|
|
37
|
+
* connection is slower than the media, and queueing would only add latency to the frames that are still fresh. */
|
|
38
|
+
export const MAX_BUFFERED_BYTES = 256 * 1024;
|
|
39
|
+
/** Reconnect delay: about a second after the first failure, doubling to a ceiling, jittered so a server restart
|
|
40
|
+
* doesn't bring every client back in the same instant. */
|
|
41
|
+
export const RELAY_BACKOFF_BASE_MS = 1_000;
|
|
42
|
+
export const RELAY_BACKOFF_MAX_MS = 15_000;
|
|
43
|
+
/** How long a socket may take to become `ready` before it is abandoned and retried (a server that accepts the
|
|
44
|
+
* upgrade but never answers the hello would otherwise hang the tier forever). */
|
|
45
|
+
export const RELAY_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
46
|
+
|
|
47
|
+
const SOCKET_OPEN = 1;
|
|
48
|
+
|
|
49
|
+
export interface RelayClientOptions {
|
|
50
|
+
url: string;
|
|
51
|
+
peerId: string;
|
|
52
|
+
createSocket: (url: string) => RelaySocket;
|
|
53
|
+
random: () => number;
|
|
54
|
+
/** Called with the sender's peer id (as stamped by the server) and the original payload. */
|
|
55
|
+
onMedia: (peerId: string, payload: Uint8Array) => void;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Views `data` as bytes when it is binary (an `ArrayBuffer` or a typed array), else `undefined`. Uses
|
|
59
|
+
* `Object.prototype.toString` rather than `instanceof` so it works for a buffer from another realm too. */
|
|
60
|
+
function toBytes(data: unknown): Uint8Array | undefined {
|
|
61
|
+
if (ArrayBuffer.isView(data)) {
|
|
62
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
63
|
+
}
|
|
64
|
+
if (Object.prototype.toString.call(data) === "[object ArrayBuffer]") {
|
|
65
|
+
return new Uint8Array(data as ArrayBuffer);
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export class RelayClient {
|
|
71
|
+
private socket: RelaySocket | undefined;
|
|
72
|
+
private closed = false;
|
|
73
|
+
private started = false;
|
|
74
|
+
private isReady = false;
|
|
75
|
+
private messageLimit = MAX_MESSAGE_BYTES;
|
|
76
|
+
private attempt = 0;
|
|
77
|
+
private wanted: string[] = [];
|
|
78
|
+
private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
|
79
|
+
private handshakeTimer: ReturnType<typeof setTimeout> | undefined;
|
|
80
|
+
private readonly decoder = new TextDecoder();
|
|
81
|
+
|
|
82
|
+
constructor(private readonly options: RelayClientOptions) {}
|
|
83
|
+
|
|
84
|
+
/** True once the server has answered the hello on the current socket, i.e. media may be sent. */
|
|
85
|
+
get ready(): boolean {
|
|
86
|
+
return this.isReady;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The largest message `sendMedia()` will send: what the server announced in `ready` (validated and clamped), or
|
|
90
|
+
* `MAX_MESSAGE_BYTES` before that, without one, and again after every disconnect until the next `ready`. The
|
|
91
|
+
* frame fragmenter sizes its fragments from this. */
|
|
92
|
+
get maxMessageBytes(): number {
|
|
93
|
+
return this.messageLimit;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Opens the socket (and keeps it open, reconnecting as needed) until `close()`. Idempotent. */
|
|
97
|
+
start(): void {
|
|
98
|
+
if (this.started || this.closed) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
this.started = true;
|
|
102
|
+
this.openSocket();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Replaces the set of senders to receive from. Sent now if the socket is ready, and again after each reconnect. */
|
|
106
|
+
setWanted(peers: Iterable<string>): void {
|
|
107
|
+
this.wanted = [...peers].slice(0, MAX_WANTED_PEERS);
|
|
108
|
+
if (this.isReady) {
|
|
109
|
+
this.sendText({ op: "want", peers: this.wanted });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Sends one media message. Returns whether it was handed to the socket - `false` (dropped) when the socket is
|
|
114
|
+
* not ready, is backed up, or the message is too large for the server to accept. */
|
|
115
|
+
sendMedia(bytes: Uint8Array): boolean {
|
|
116
|
+
const socket = this.socket;
|
|
117
|
+
if (
|
|
118
|
+
!socket ||
|
|
119
|
+
!this.isReady ||
|
|
120
|
+
socket.readyState !== SOCKET_OPEN ||
|
|
121
|
+
socket.bufferedAmount > MAX_BUFFERED_BYTES ||
|
|
122
|
+
bytes.length > this.messageLimit
|
|
123
|
+
) {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
socket.send(bytes);
|
|
128
|
+
return true;
|
|
129
|
+
} catch {
|
|
130
|
+
// The close that follows a dead socket reconnects.
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Closes the socket and stops reconnecting for good. Idempotent. */
|
|
136
|
+
close(): void {
|
|
137
|
+
if (this.closed) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
this.closed = true;
|
|
141
|
+
this.isReady = false;
|
|
142
|
+
this.messageLimit = MAX_MESSAGE_BYTES;
|
|
143
|
+
clearTimeout(this.reconnectTimer);
|
|
144
|
+
clearTimeout(this.handshakeTimer);
|
|
145
|
+
this.reconnectTimer = this.handshakeTimer = undefined;
|
|
146
|
+
this.detach();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private openSocket(): void {
|
|
150
|
+
let socket: RelaySocket;
|
|
151
|
+
try {
|
|
152
|
+
socket = this.options.createSocket(this.options.url);
|
|
153
|
+
} catch {
|
|
154
|
+
this.scheduleReconnect();
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
this.socket = socket;
|
|
158
|
+
socket.binaryType = "arraybuffer";
|
|
159
|
+
socket.onopen = () => this.sendText({ op: "hello", v: 1, peer: this.options.peerId });
|
|
160
|
+
socket.onmessage = (event) => this.handleMessage(socket, event.data);
|
|
161
|
+
socket.onclose = () => this.handleClose(socket);
|
|
162
|
+
socket.onerror = () => undefined;
|
|
163
|
+
this.handshakeTimer = setTimeout(() => {
|
|
164
|
+
this.handshakeTimer = undefined;
|
|
165
|
+
// Give up on this attempt as if the socket had closed; the close event of a socket we abandon is ignored.
|
|
166
|
+
this.handleClose(socket);
|
|
167
|
+
}, RELAY_HANDSHAKE_TIMEOUT_MS);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private handleMessage(socket: RelaySocket, data: unknown): void {
|
|
171
|
+
if (socket !== this.socket) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (typeof data === "string") {
|
|
175
|
+
this.handleText(data);
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
const bytes = toBytes(data);
|
|
179
|
+
if (!bytes || !this.isReady) {
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const idLength = bytes.length > 0 ? bytes[0] : 0;
|
|
183
|
+
if (idLength === 0 || bytes.length < 1 + idLength) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
// A malformed id decodes to replacement characters, which match no receiver, so it needs no separate check.
|
|
187
|
+
const peerId = this.decoder.decode(bytes.subarray(1, 1 + idLength));
|
|
188
|
+
this.options.onMedia(peerId, bytes.subarray(1 + idLength));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private handleText(text: string): void {
|
|
192
|
+
let message: unknown;
|
|
193
|
+
try {
|
|
194
|
+
message = JSON.parse(text);
|
|
195
|
+
} catch {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (!message || typeof message !== "object" || (message as { op?: unknown }).op !== "ready" || this.isReady) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
this.isReady = true;
|
|
202
|
+
this.attempt = 0;
|
|
203
|
+
const announced = (message as { maxMessageBytes?: unknown }).maxMessageBytes;
|
|
204
|
+
if (typeof announced === "number" && Number.isInteger(announced) && announced >= MIN_NEGOTIATED_MESSAGE_BYTES) {
|
|
205
|
+
this.messageLimit = Math.min(announced, MAX_NEGOTIATED_MESSAGE_BYTES);
|
|
206
|
+
}
|
|
207
|
+
clearTimeout(this.handshakeTimer);
|
|
208
|
+
this.handshakeTimer = undefined;
|
|
209
|
+
// The server's wanted set is per socket, so it starts empty on every (re)connect.
|
|
210
|
+
this.sendText({ op: "want", peers: this.wanted });
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private handleClose(socket: RelaySocket): void {
|
|
214
|
+
if (socket !== this.socket) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
this.isReady = false;
|
|
218
|
+
// The next server might be a different one (a rolling upgrade), so the limit is learned afresh.
|
|
219
|
+
this.messageLimit = MAX_MESSAGE_BYTES;
|
|
220
|
+
clearTimeout(this.handshakeTimer);
|
|
221
|
+
this.handshakeTimer = undefined;
|
|
222
|
+
this.detach();
|
|
223
|
+
this.scheduleReconnect();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Detaches and closes the current socket, if any, so its late events can't reach this client. */
|
|
227
|
+
private detach(): void {
|
|
228
|
+
const socket = this.socket;
|
|
229
|
+
this.socket = undefined;
|
|
230
|
+
if (!socket) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
socket.onopen = socket.onmessage = socket.onclose = socket.onerror = null;
|
|
234
|
+
try {
|
|
235
|
+
socket.close(1000, "closing");
|
|
236
|
+
} catch {
|
|
237
|
+
// Already gone.
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private scheduleReconnect(): void {
|
|
242
|
+
const ceiling = Math.min(RELAY_BACKOFF_MAX_MS, RELAY_BACKOFF_BASE_MS * 2 ** Math.min(this.attempt, 10));
|
|
243
|
+
const delay = ceiling / 2 + (this.options.random() * ceiling) / 2;
|
|
244
|
+
this.attempt += 1;
|
|
245
|
+
this.reconnectTimer = setTimeout(() => {
|
|
246
|
+
this.reconnectTimer = undefined;
|
|
247
|
+
this.openSocket();
|
|
248
|
+
}, delay);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
private sendText(message: object): void {
|
|
252
|
+
const socket = this.socket;
|
|
253
|
+
if (!socket || socket.readyState !== SOCKET_OPEN) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
try {
|
|
257
|
+
socket.send(JSON.stringify(message));
|
|
258
|
+
} catch {
|
|
259
|
+
// The close that follows a dead socket reconnects and re-sends.
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
import { AudioPlayer } from "./AudioPlayer.js";
|
|
6
|
+
import { decodeFragment, FrameReassembler, KIND_AUDIO } from "./frames.js";
|
|
7
|
+
import type { AudioContextLike, RelayEnv } from "./relayEnv.js";
|
|
8
|
+
import { VideoPlayer } from "./VideoPlayer.js";
|
|
9
|
+
|
|
10
|
+
/** The canvas is sized to the first decoded picture; until then it is this size. */
|
|
11
|
+
const INITIAL_CANVAS_WIDTH = 320;
|
|
12
|
+
const INITIAL_CANVAS_HEIGHT = 240;
|
|
13
|
+
/** The frame rate `captureStream()` samples the canvas at, matching what the sender produces. */
|
|
14
|
+
const CAPTURE_FPS = 15;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* One remote participant as seen through the relay: reassembles their fragments, decodes them, and exposes the result
|
|
18
|
+
* as an ordinary `MediaStream` (`stream`) so the UI plays a relayed peer exactly like a WebRTC one.
|
|
19
|
+
*
|
|
20
|
+
* The stream exists, with both tracks, from construction - silent and blank until media arrives - so the caller can
|
|
21
|
+
* attach it to an element straight away and never has to swap a stream in later. Audio goes through a
|
|
22
|
+
* `MediaStreamAudioDestinationNode` on the shared playback context; video is drawn to a canvas and captured.
|
|
23
|
+
*/
|
|
24
|
+
export class RelayReceiver {
|
|
25
|
+
readonly stream: MediaStream;
|
|
26
|
+
private readonly audioFrames = new FrameReassembler();
|
|
27
|
+
private readonly videoFrames = new FrameReassembler();
|
|
28
|
+
private readonly audio: AudioPlayer;
|
|
29
|
+
private readonly video: VideoPlayer | undefined;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
env: RelayEnv,
|
|
33
|
+
playbackContext: AudioContextLike,
|
|
34
|
+
) {
|
|
35
|
+
const destination = playbackContext.createMediaStreamDestination();
|
|
36
|
+
this.audio = new AudioPlayer(env, playbackContext, destination);
|
|
37
|
+
|
|
38
|
+
const canvas = env.createCanvas();
|
|
39
|
+
canvas.width = INITIAL_CANVAS_WIDTH;
|
|
40
|
+
canvas.height = INITIAL_CANVAS_HEIGHT;
|
|
41
|
+
const context = canvas.getContext("2d");
|
|
42
|
+
this.video = context ? new VideoPlayer(env, canvas, context) : undefined;
|
|
43
|
+
|
|
44
|
+
this.stream = env.createMediaStream([...destination.stream.getAudioTracks(), ...canvas.captureStream(CAPTURE_FPS).getVideoTracks()]);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Handles one relayed payload from this sender (a media fragment). Anything malformed is ignored. */
|
|
48
|
+
handleMedia(payload: Uint8Array): void {
|
|
49
|
+
const fragment = decodeFragment(payload);
|
|
50
|
+
if (!fragment) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const frame = (fragment.kind === KIND_AUDIO ? this.audioFrames : this.videoFrames).push(fragment);
|
|
54
|
+
if (!frame) {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (frame.kind === KIND_AUDIO) {
|
|
58
|
+
this.audio.push(frame);
|
|
59
|
+
} else {
|
|
60
|
+
this.video?.push(frame);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Releases the decoders and ends the stream's tracks. */
|
|
65
|
+
close(): void {
|
|
66
|
+
this.audio.close();
|
|
67
|
+
this.video?.close();
|
|
68
|
+
for (const track of this.stream.getTracks()) {
|
|
69
|
+
track.stop();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
import { AudioSender } from "./AudioSender.js";
|
|
6
|
+
import { DEFAULT_MESSAGE_BYTES, fragmentFrame, type FrameKind, KIND_AUDIO, KIND_VIDEO } from "./frames.js";
|
|
7
|
+
import type { RelayEnv } from "./relayEnv.js";
|
|
8
|
+
import type { PublishFrame } from "./senderCommon.js";
|
|
9
|
+
import { VideoSender } from "./VideoSender.js";
|
|
10
|
+
|
|
11
|
+
const SEQ_MODULUS = 0x10000;
|
|
12
|
+
|
|
13
|
+
/** Numbers and fragments the frames of one sender. The counters belong to the transport rather than to a capture
|
|
14
|
+
* session, so stopping and restarting sending never resets `seq` - a receiver still holding the old value would
|
|
15
|
+
* otherwise treat the restarted stream as older than what it has already seen. */
|
|
16
|
+
export class FramePublisher {
|
|
17
|
+
private readonly seqs: Record<FrameKind, number> = { [KIND_AUDIO]: 0, [KIND_VIDEO]: 0 };
|
|
18
|
+
|
|
19
|
+
/** `send` returns whether one message was accepted by the socket. */
|
|
20
|
+
/** `maxMessageBytes` is the largest message the socket may send right now (it changes when the server's `ready`
|
|
21
|
+
* announces a limit), so it is read for every frame rather than fixed at construction. */
|
|
22
|
+
constructor(
|
|
23
|
+
private readonly send: (message: Uint8Array) => boolean,
|
|
24
|
+
private readonly maxMessageBytes: () => number = () => DEFAULT_MESSAGE_BYTES,
|
|
25
|
+
) {}
|
|
26
|
+
|
|
27
|
+
/** Fragments and sends one frame, stopping at the first fragment the socket refuses (the rest could not be
|
|
28
|
+
* used without it). Returns whether the whole frame went out. */
|
|
29
|
+
publish: PublishFrame = (kind, keyFrame, data, timestampMs) => {
|
|
30
|
+
const seq = this.seqs[kind];
|
|
31
|
+
this.seqs[kind] = (seq + 1) % SEQ_MODULUS;
|
|
32
|
+
const fragments = fragmentFrame(kind, keyFrame, seq, timestampMs, data, this.maxMessageBytes());
|
|
33
|
+
if (fragments.length === 0) {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
for (const fragment of fragments) {
|
|
37
|
+
if (!this.send(fragment)) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return true;
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RelaySenderOptions {
|
|
46
|
+
env: RelayEnv;
|
|
47
|
+
send: (message: Uint8Array) => boolean;
|
|
48
|
+
canSend: () => boolean;
|
|
49
|
+
/** The socket's current message limit; defaults to what an older server accepts. */
|
|
50
|
+
maxMessageBytes?: () => number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The publishing half of the relay: the two capture paths plus the frame numbering they share. */
|
|
54
|
+
export class RelaySender {
|
|
55
|
+
private readonly audio: AudioSender;
|
|
56
|
+
private readonly video: VideoSender;
|
|
57
|
+
|
|
58
|
+
constructor(options: RelaySenderOptions) {
|
|
59
|
+
const deps = { env: options.env, publish: new FramePublisher(options.send, options.maxMessageBytes).publish, canSend: options.canSend };
|
|
60
|
+
this.audio = new AudioSender(deps);
|
|
61
|
+
this.video = new VideoSender(deps);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Starts or stops encoding whatever tracks are set. Stopping releases the encoders and audio nodes. */
|
|
65
|
+
setActive(active: boolean): void {
|
|
66
|
+
this.audio.setActive(active);
|
|
67
|
+
this.video.setActive(active);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** `null` sends nothing for that kind. Safe before or after `setActive`. */
|
|
71
|
+
setTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void {
|
|
72
|
+
(kind === "audio" ? this.audio : this.video).setTrack(track);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* The last-resort media tier of a call: when two participants' WebRTC connection cannot be made at all (typically a
|
|
7
|
+
* network that blocks UDP and TURN alike), their audio and video travel over a WebSocket to the server, which fans
|
|
8
|
+
* each participant's frames out to whoever asked for them. This module is the facade the mesh manager talks to; the
|
|
9
|
+
* pieces behind it are:
|
|
10
|
+
*
|
|
11
|
+
* - `RelayClient` - the socket: hello/ready/want, reconnection, dropping media it cannot send promptly.
|
|
12
|
+
* - `frames` - the fragment format inside each WebSocket message and the reassembly of it.
|
|
13
|
+
* - `RelaySender` (`AudioSender`, `VideoSender`) - capture and encode the local tracks (Opus and VP8 via WebCodecs).
|
|
14
|
+
* - `RelayReceiver` (`AudioPlayer`, `VideoPlayer`) - decode a remote participant back into a `MediaStream`.
|
|
15
|
+
*
|
|
16
|
+
* There is one transport (one socket) per meeting, however many peers use it, and the codecs are fixed - there is no
|
|
17
|
+
* negotiation, because both ends are this same code.
|
|
18
|
+
*/
|
|
19
|
+
import { apiOrigin } from "@rapidmx/react-shared/util/api.js";
|
|
20
|
+
import { PLAYBACK_SAMPLE_RATE } from "./AudioPlayer.js";
|
|
21
|
+
import { resumeAudioContext } from "./audioResume.js";
|
|
22
|
+
import { RelayClient } from "./RelayClient.js";
|
|
23
|
+
import { detectRelayEnv, type AudioContextLike, type RelayEnv } from "./relayEnv.js";
|
|
24
|
+
import { RelayReceiver } from "./RelayReceiver.js";
|
|
25
|
+
import { RelaySender } from "./RelaySender.js";
|
|
26
|
+
|
|
27
|
+
export interface RelayTransport {
|
|
28
|
+
/** false when this browser lacks WebSocket, AudioEncoder/AudioDecoder/VideoEncoder/VideoDecoder, VideoFrame,
|
|
29
|
+
* AudioContext or canvas.captureStream - the mesh then never falls back to the relay. */
|
|
30
|
+
readonly supported: boolean;
|
|
31
|
+
/** Start receiving `peerId`'s media (opens/keeps the socket, adds to `want`). `onStream` gets a MediaStream that
|
|
32
|
+
* plays that peer (one audio + one video track). Called at most once per stream creation; calling receiveFrom
|
|
33
|
+
* again for the same peer is a no-op. */
|
|
34
|
+
receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void;
|
|
35
|
+
stopReceivingFrom(peerId: string): void;
|
|
36
|
+
/** Start/stop publishing the local tracks (encode + send). Only encodes while active; stopping releases the
|
|
37
|
+
* encoders and audio nodes. */
|
|
38
|
+
setSending(active: boolean): void;
|
|
39
|
+
/** Same semantics as `MeshConnectionManager.setLocalTrack`: null = send nothing for that kind. Safe to call
|
|
40
|
+
* before or after `setSending`. */
|
|
41
|
+
setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
|
|
42
|
+
/** Closes the socket, all coders, streams and timers. Idempotent. */
|
|
43
|
+
close(): void;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface RelayTransportOptions {
|
|
47
|
+
meetingUid: string;
|
|
48
|
+
/** This participant's peer id on the meeting's signaling channel - the id the server stamps on what it sends. */
|
|
49
|
+
peerId: string;
|
|
50
|
+
/** Overrides the socket URL (default: `relayUrl(meetingUid)`). */
|
|
51
|
+
url?: string;
|
|
52
|
+
/** Overrides every browser API the relay uses (default: `detectRelayEnv()`), so a test can drive it with fakes. */
|
|
53
|
+
env?: RelayEnv;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The relay WebSocket's URL for a meeting: this deployment's origin (the API origin when the UI is served from a
|
|
57
|
+
* different one, else the page's own), `http` becoming `ws` - the same derivation `pushUrl()` uses for `/push`.
|
|
58
|
+
* `undefined` when there is no origin at all (not running in a browser). */
|
|
59
|
+
export function relayUrl(meetingUid: string): string | undefined {
|
|
60
|
+
const origin = apiOrigin() || (typeof window !== "undefined" ? window.location.origin : "");
|
|
61
|
+
return origin ? `${origin.replace(/^http/i, "ws")}/api/mail/video-meetings/relay/${encodeURIComponent(meetingUid)}` : undefined;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** What `createRelayTransport()` returns on a browser that cannot run the relay: every call is a no-op. */
|
|
65
|
+
const UNSUPPORTED_TRANSPORT: RelayTransport = {
|
|
66
|
+
supported: false,
|
|
67
|
+
receiveFrom: () => undefined,
|
|
68
|
+
stopReceivingFrom: () => undefined,
|
|
69
|
+
setSending: () => undefined,
|
|
70
|
+
setLocalTrack: () => undefined,
|
|
71
|
+
close: () => undefined,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
class WebSocketRelayTransport implements RelayTransport {
|
|
75
|
+
readonly supported = true;
|
|
76
|
+
private readonly client: RelayClient;
|
|
77
|
+
private readonly sender: RelaySender;
|
|
78
|
+
private readonly receivers = new Map<string, RelayReceiver>();
|
|
79
|
+
private playback: AudioContextLike | undefined;
|
|
80
|
+
private stopResume: (() => void) | undefined;
|
|
81
|
+
private closed = false;
|
|
82
|
+
|
|
83
|
+
constructor(
|
|
84
|
+
private readonly env: RelayEnv,
|
|
85
|
+
url: string,
|
|
86
|
+
peerId: string,
|
|
87
|
+
) {
|
|
88
|
+
this.client = new RelayClient({
|
|
89
|
+
url,
|
|
90
|
+
peerId,
|
|
91
|
+
createSocket: (target) => env.createSocket(target),
|
|
92
|
+
random: () => env.random(),
|
|
93
|
+
onMedia: (sender, payload) => this.receivers.get(sender)?.handleMedia(payload),
|
|
94
|
+
});
|
|
95
|
+
this.sender = new RelaySender({
|
|
96
|
+
env,
|
|
97
|
+
send: (message) => this.client.sendMedia(message),
|
|
98
|
+
canSend: () => this.client.ready,
|
|
99
|
+
maxMessageBytes: () => this.client.maxMessageBytes,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void {
|
|
104
|
+
if (this.closed || this.receivers.has(peerId)) {
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const receiver = this.createReceiver();
|
|
108
|
+
if (!receiver) {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
this.receivers.set(peerId, receiver);
|
|
112
|
+
this.client.start();
|
|
113
|
+
this.client.setWanted(this.receivers.keys());
|
|
114
|
+
onStream(receiver.stream);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
stopReceivingFrom(peerId: string): void {
|
|
118
|
+
const receiver = this.receivers.get(peerId);
|
|
119
|
+
if (!receiver) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
this.receivers.delete(peerId);
|
|
123
|
+
receiver.close();
|
|
124
|
+
this.client.setWanted(this.receivers.keys());
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
setSending(active: boolean): void {
|
|
128
|
+
if (this.closed) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (active) {
|
|
132
|
+
this.client.start();
|
|
133
|
+
}
|
|
134
|
+
this.sender.setActive(active);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void {
|
|
138
|
+
if (!this.closed) {
|
|
139
|
+
this.sender.setTrack(kind, track);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
close(): void {
|
|
144
|
+
if (this.closed) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.closed = true;
|
|
148
|
+
this.sender.setActive(false);
|
|
149
|
+
for (const receiver of this.receivers.values()) {
|
|
150
|
+
receiver.close();
|
|
151
|
+
}
|
|
152
|
+
this.receivers.clear();
|
|
153
|
+
this.client.close();
|
|
154
|
+
this.stopResume?.();
|
|
155
|
+
// `close()` rejects for a context that is already closed; nothing to do about it.
|
|
156
|
+
this.playback?.close().catch(() => undefined);
|
|
157
|
+
this.playback = undefined;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** A receiver on the shared playback context, or `undefined` if this browser will not build one (the peer then
|
|
161
|
+
* simply has no relayed media). */
|
|
162
|
+
private createReceiver(): RelayReceiver | undefined {
|
|
163
|
+
try {
|
|
164
|
+
if (!this.playback) {
|
|
165
|
+
this.playback = this.env.createAudioContext(PLAYBACK_SAMPLE_RATE);
|
|
166
|
+
// Nothing the user just did created this context, so it may start suspended - see `resumeAudioContext`.
|
|
167
|
+
this.stopResume = resumeAudioContext(this.playback, this.env.document);
|
|
168
|
+
}
|
|
169
|
+
return new RelayReceiver(this.env, this.playback);
|
|
170
|
+
} catch {
|
|
171
|
+
return undefined;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Creates the relay transport for one meeting. Returns an inert transport (`supported: false`) when the browser lacks
|
|
178
|
+
* something the relay needs, so the caller can simply check `supported` once and never fall back to it.
|
|
179
|
+
* No socket is opened until the first `receiveFrom()` or `setSending(true)`.
|
|
180
|
+
*/
|
|
181
|
+
export function createRelayTransport(options: RelayTransportOptions): RelayTransport {
|
|
182
|
+
const env = options.env ?? detectRelayEnv();
|
|
183
|
+
const url = options.url ?? relayUrl(options.meetingUid);
|
|
184
|
+
if (!env || !url) {
|
|
185
|
+
return UNSUPPORTED_TRANSPORT;
|
|
186
|
+
}
|
|
187
|
+
return new WebSocketRelayTransport(env, url, options.peerId);
|
|
188
|
+
}
|