@rapidmx/meet-plugin 0.5.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +15 -2
  2. package/apps/meet/[token].tsx +3 -2
  3. package/apps/meet/_CallControls.tsx +28 -2
  4. package/apps/meet/_CallView.tsx +23 -2
  5. package/apps/meet/_EffectsPanel.tsx +134 -0
  6. package/apps/meet/_MeetLobby.tsx +22 -0
  7. package/apps/meet/_ParticipantTile.tsx +62 -9
  8. package/apps/meet/_icons.tsx +4 -0
  9. package/apps/meet/_meetApi.ts +6 -0
  10. package/apps/shared/media/filters/VideoFilterProcessor.ts +383 -0
  11. package/apps/shared/media/filters/accessories.ts +262 -0
  12. package/apps/shared/media/filters/backgroundImage.ts +69 -0
  13. package/apps/shared/media/filters/filterTypes.ts +89 -0
  14. package/apps/shared/media/filters/mlModels.ts +174 -0
  15. package/apps/shared/media/filters/pixelEffects.ts +61 -0
  16. package/apps/shared/media/mediaPreferences.ts +105 -0
  17. package/apps/shared/media/useLocalMedia.ts +198 -14
  18. package/apps/shared/relay/AudioPlayer.ts +111 -0
  19. package/apps/shared/relay/AudioSender.ts +194 -0
  20. package/apps/shared/relay/RelayClient.ts +262 -0
  21. package/apps/shared/relay/RelayReceiver.ts +72 -0
  22. package/apps/shared/relay/RelaySender.ts +74 -0
  23. package/apps/shared/relay/RelayTransport.ts +188 -0
  24. package/apps/shared/relay/VideoPlayer.ts +110 -0
  25. package/apps/shared/relay/VideoSender.ts +216 -0
  26. package/apps/shared/relay/audioResume.ts +55 -0
  27. package/apps/shared/relay/frames.ts +275 -0
  28. package/apps/shared/relay/relayEnv.ts +230 -0
  29. package/apps/shared/relay/senderCommon.ts +30 -0
  30. package/apps/shared/webrtc/MeshConnectionManager.ts +158 -6
  31. package/apps/shared/webrtc/realPeerConnection.ts +22 -0
  32. package/apps/shared/webrtc/types.ts +32 -2
  33. package/dist/apps/meet/[token].js +3 -3
  34. package/dist/apps/meet/_CallControls.d.ts +2 -1
  35. package/dist/apps/meet/_CallControls.js +6 -3
  36. package/dist/apps/meet/_CallView.d.ts +4 -1
  37. package/dist/apps/meet/_CallView.js +17 -3
  38. package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
  39. package/dist/apps/meet/_EffectsPanel.js +60 -0
  40. package/dist/apps/meet/_MeetLobby.d.ts +4 -0
  41. package/dist/apps/meet/_MeetLobby.js +8 -1
  42. package/dist/apps/meet/_ParticipantTile.d.ts +7 -1
  43. package/dist/apps/meet/_ParticipantTile.js +20 -3
  44. package/dist/apps/meet/_icons.d.ts +1 -0
  45. package/dist/apps/meet/_icons.js +1 -0
  46. package/dist/apps/meet/_meetApi.d.ts +6 -0
  47. package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +75 -0
  48. package/dist/apps/shared/media/filters/VideoFilterProcessor.js +304 -0
  49. package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
  50. package/dist/apps/shared/media/filters/accessories.js +211 -0
  51. package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
  52. package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
  53. package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
  54. package/dist/apps/shared/media/filters/filterTypes.js +63 -0
  55. package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
  56. package/dist/apps/shared/media/filters/mlModels.js +106 -0
  57. package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
  58. package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
  59. package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
  60. package/dist/apps/shared/media/mediaPreferences.js +93 -0
  61. package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
  62. package/dist/apps/shared/media/useLocalMedia.js +160 -14
  63. package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
  64. package/dist/apps/shared/relay/AudioPlayer.js +103 -0
  65. package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
  66. package/dist/apps/shared/relay/AudioSender.js +172 -0
  67. package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
  68. package/dist/apps/shared/relay/RelayClient.js +236 -0
  69. package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
  70. package/dist/apps/shared/relay/RelayReceiver.js +59 -0
  71. package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
  72. package/dist/apps/shared/relay/RelaySender.js +54 -0
  73. package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
  74. package/dist/apps/shared/relay/RelayTransport.js +142 -0
  75. package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
  76. package/dist/apps/shared/relay/VideoPlayer.js +102 -0
  77. package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
  78. package/dist/apps/shared/relay/VideoSender.js +204 -0
  79. package/dist/apps/shared/relay/audioResume.d.ts +12 -0
  80. package/dist/apps/shared/relay/audioResume.js +45 -0
  81. package/dist/apps/shared/relay/frames.d.ts +101 -0
  82. package/dist/apps/shared/relay/frames.js +221 -0
  83. package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
  84. package/dist/apps/shared/relay/relayEnv.js +69 -0
  85. package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
  86. package/dist/apps/shared/relay/senderCommon.js +10 -0
  87. package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +51 -1
  88. package/dist/apps/shared/webrtc/MeshConnectionManager.js +137 -6
  89. package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
  90. package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
  91. package/dist/apps/shared/webrtc/types.d.ts +29 -2
  92. package/dist/lib/index.js +2 -0
  93. package/dist/lib/index.js.map +1 -1
  94. package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
  95. package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
  96. package/dist/lib/util/EffectsUtils.js +21 -0
  97. package/dist/lib/util/EffectsUtils.js.map +1 -0
  98. package/dist/lib/util/IceServerUtils.js +25 -4
  99. package/dist/lib/util/IceServerUtils.js.map +1 -1
  100. package/dist/lib/util/RedisRelayBus.js +241 -0
  101. package/dist/lib/util/RedisRelayBus.js.map +1 -0
  102. package/dist/lib/util/RelayHub.js +405 -0
  103. package/dist/lib/util/RelayHub.js.map +1 -0
  104. package/dist/types/index.d.ts +2 -0
  105. package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
  106. package/dist/types/util/EffectsUtils.d.ts +10 -0
  107. package/dist/types/util/IceServerUtils.d.ts +12 -1
  108. package/dist/types/util/RedisRelayBus.d.ts +131 -0
  109. package/dist/types/util/RelayHub.d.ts +188 -0
  110. package/package.json +24 -7
@@ -0,0 +1,65 @@
1
+ import type { RelaySocket } from "./relayEnv.js";
2
+ /** The largest binary message an older server accepts - it closes the socket on a larger one (uWS's default limit).
3
+ * Assumed until the server's `ready` announces a different `maxMessageBytes`. */
4
+ export declare const MAX_MESSAGE_BYTES = 16384;
5
+ /** A `maxMessageBytes` below this is not believed (nothing useful fits), and one above
6
+ * `MAX_NEGOTIATED_MESSAGE_BYTES` is clamped to it. */
7
+ export declare const MIN_NEGOTIATED_MESSAGE_BYTES = 1024;
8
+ export declare const MAX_NEGOTIATED_MESSAGE_BYTES: number;
9
+ /** The most senders the server lets one socket `want`. */
10
+ export declare const MAX_WANTED_PEERS = 32;
11
+ /** A media send is dropped, not queued, while more than this many bytes are still waiting to leave - the
12
+ * connection is slower than the media, and queueing would only add latency to the frames that are still fresh. */
13
+ export declare const MAX_BUFFERED_BYTES: number;
14
+ /** Reconnect delay: about a second after the first failure, doubling to a ceiling, jittered so a server restart
15
+ * doesn't bring every client back in the same instant. */
16
+ export declare const RELAY_BACKOFF_BASE_MS = 1000;
17
+ export declare const RELAY_BACKOFF_MAX_MS = 15000;
18
+ /** How long a socket may take to become `ready` before it is abandoned and retried (a server that accepts the
19
+ * upgrade but never answers the hello would otherwise hang the tier forever). */
20
+ export declare const RELAY_HANDSHAKE_TIMEOUT_MS = 10000;
21
+ export interface RelayClientOptions {
22
+ url: string;
23
+ peerId: string;
24
+ createSocket: (url: string) => RelaySocket;
25
+ random: () => number;
26
+ /** Called with the sender's peer id (as stamped by the server) and the original payload. */
27
+ onMedia: (peerId: string, payload: Uint8Array) => void;
28
+ }
29
+ export declare class RelayClient {
30
+ private readonly options;
31
+ private socket;
32
+ private closed;
33
+ private started;
34
+ private isReady;
35
+ private messageLimit;
36
+ private attempt;
37
+ private wanted;
38
+ private reconnectTimer;
39
+ private handshakeTimer;
40
+ private readonly decoder;
41
+ constructor(options: RelayClientOptions);
42
+ /** True once the server has answered the hello on the current socket, i.e. media may be sent. */
43
+ get ready(): boolean;
44
+ /** The largest message `sendMedia()` will send: what the server announced in `ready` (validated and clamped), or
45
+ * `MAX_MESSAGE_BYTES` before that, without one, and again after every disconnect until the next `ready`. The
46
+ * frame fragmenter sizes its fragments from this. */
47
+ get maxMessageBytes(): number;
48
+ /** Opens the socket (and keeps it open, reconnecting as needed) until `close()`. Idempotent. */
49
+ start(): void;
50
+ /** Replaces the set of senders to receive from. Sent now if the socket is ready, and again after each reconnect. */
51
+ setWanted(peers: Iterable<string>): void;
52
+ /** Sends one media message. Returns whether it was handed to the socket - `false` (dropped) when the socket is
53
+ * not ready, is backed up, or the message is too large for the server to accept. */
54
+ sendMedia(bytes: Uint8Array): boolean;
55
+ /** Closes the socket and stops reconnecting for good. Idempotent. */
56
+ close(): void;
57
+ private openSocket;
58
+ private handleMessage;
59
+ private handleText;
60
+ private handleClose;
61
+ /** Detaches and closes the current socket, if any, so its late events can't reach this client. */
62
+ private detach;
63
+ private scheduleReconnect;
64
+ private sendText;
65
+ }
@@ -0,0 +1,236 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ /**
6
+ * The WebSocket to the server's media relay - the transport under the last-resort tier of a call, used for a pair of
7
+ * participants whose WebRTC connection could not be made to work at all.
8
+ *
9
+ * ## Wire protocol (fixed by the server)
10
+ *
11
+ * - The first message is text JSON `{"op":"hello","v":1,"peer":"<peerId>"}`; the server answers `{"op":"ready","v":1}`,
12
+ * plus `"maxMessageBytes"` - the largest client-to-server message it accepts - on a current server (absent on an
13
+ * older one, which accepts `MAX_MESSAGE_BYTES`). Nothing else may be sent before `ready`.
14
+ * - `{"op":"want","peers":[...]}` REPLACES the set of senders this socket receives from (the server accepts at most
15
+ * `MAX_WANTED_PEERS`). It has to be re-sent after every reconnect, because the server's set dies with the socket.
16
+ * - A binary message from the client is an opaque payload of at most `maxMessageBytes`. The server forwards it to
17
+ * every socket that wants this sender as `[1 byte N][N bytes UTF-8 sender peer id][payload]`, stamping the sender
18
+ * itself so a peer cannot speak as another.
19
+ * - Binary messages may be dropped under backpressure and the socket may close at any moment, so this client
20
+ * reconnects with exponential backoff and treats every media send as best effort.
21
+ *
22
+ * Authentication is the `jwt` cookie already in place (see `GuestSignalingClient`), so this client sets no cookie.
23
+ */
24
+ import { DEFAULT_MESSAGE_BYTES } from "./frames.js";
25
+ /** The largest binary message an older server accepts - it closes the socket on a larger one (uWS's default limit).
26
+ * Assumed until the server's `ready` announces a different `maxMessageBytes`. */
27
+ export const MAX_MESSAGE_BYTES = DEFAULT_MESSAGE_BYTES;
28
+ /** A `maxMessageBytes` below this is not believed (nothing useful fits), and one above
29
+ * `MAX_NEGOTIATED_MESSAGE_BYTES` is clamped to it. */
30
+ export const MIN_NEGOTIATED_MESSAGE_BYTES = 1024;
31
+ export const MAX_NEGOTIATED_MESSAGE_BYTES = 1024 * 1024;
32
+ /** The most senders the server lets one socket `want`. */
33
+ export const MAX_WANTED_PEERS = 32;
34
+ /** A media send is dropped, not queued, while more than this many bytes are still waiting to leave - the
35
+ * connection is slower than the media, and queueing would only add latency to the frames that are still fresh. */
36
+ export const MAX_BUFFERED_BYTES = 256 * 1024;
37
+ /** Reconnect delay: about a second after the first failure, doubling to a ceiling, jittered so a server restart
38
+ * doesn't bring every client back in the same instant. */
39
+ export const RELAY_BACKOFF_BASE_MS = 1000;
40
+ export const RELAY_BACKOFF_MAX_MS = 15000;
41
+ /** How long a socket may take to become `ready` before it is abandoned and retried (a server that accepts the
42
+ * upgrade but never answers the hello would otherwise hang the tier forever). */
43
+ export const RELAY_HANDSHAKE_TIMEOUT_MS = 10000;
44
+ const SOCKET_OPEN = 1;
45
+ /** Views `data` as bytes when it is binary (an `ArrayBuffer` or a typed array), else `undefined`. Uses
46
+ * `Object.prototype.toString` rather than `instanceof` so it works for a buffer from another realm too. */
47
+ function toBytes(data) {
48
+ if (ArrayBuffer.isView(data)) {
49
+ return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
50
+ }
51
+ if (Object.prototype.toString.call(data) === "[object ArrayBuffer]") {
52
+ return new Uint8Array(data);
53
+ }
54
+ return undefined;
55
+ }
56
+ export class RelayClient {
57
+ constructor(options) {
58
+ this.options = options;
59
+ this.closed = false;
60
+ this.started = false;
61
+ this.isReady = false;
62
+ this.messageLimit = MAX_MESSAGE_BYTES;
63
+ this.attempt = 0;
64
+ this.wanted = [];
65
+ this.decoder = new TextDecoder();
66
+ }
67
+ /** True once the server has answered the hello on the current socket, i.e. media may be sent. */
68
+ get ready() {
69
+ return this.isReady;
70
+ }
71
+ /** The largest message `sendMedia()` will send: what the server announced in `ready` (validated and clamped), or
72
+ * `MAX_MESSAGE_BYTES` before that, without one, and again after every disconnect until the next `ready`. The
73
+ * frame fragmenter sizes its fragments from this. */
74
+ get maxMessageBytes() {
75
+ return this.messageLimit;
76
+ }
77
+ /** Opens the socket (and keeps it open, reconnecting as needed) until `close()`. Idempotent. */
78
+ start() {
79
+ if (this.started || this.closed) {
80
+ return;
81
+ }
82
+ this.started = true;
83
+ this.openSocket();
84
+ }
85
+ /** Replaces the set of senders to receive from. Sent now if the socket is ready, and again after each reconnect. */
86
+ setWanted(peers) {
87
+ this.wanted = [...peers].slice(0, MAX_WANTED_PEERS);
88
+ if (this.isReady) {
89
+ this.sendText({ op: "want", peers: this.wanted });
90
+ }
91
+ }
92
+ /** Sends one media message. Returns whether it was handed to the socket - `false` (dropped) when the socket is
93
+ * not ready, is backed up, or the message is too large for the server to accept. */
94
+ sendMedia(bytes) {
95
+ const socket = this.socket;
96
+ if (!socket ||
97
+ !this.isReady ||
98
+ socket.readyState !== SOCKET_OPEN ||
99
+ socket.bufferedAmount > MAX_BUFFERED_BYTES ||
100
+ bytes.length > this.messageLimit) {
101
+ return false;
102
+ }
103
+ try {
104
+ socket.send(bytes);
105
+ return true;
106
+ }
107
+ catch {
108
+ // The close that follows a dead socket reconnects.
109
+ return false;
110
+ }
111
+ }
112
+ /** Closes the socket and stops reconnecting for good. Idempotent. */
113
+ close() {
114
+ if (this.closed) {
115
+ return;
116
+ }
117
+ this.closed = true;
118
+ this.isReady = false;
119
+ this.messageLimit = MAX_MESSAGE_BYTES;
120
+ clearTimeout(this.reconnectTimer);
121
+ clearTimeout(this.handshakeTimer);
122
+ this.reconnectTimer = this.handshakeTimer = undefined;
123
+ this.detach();
124
+ }
125
+ openSocket() {
126
+ let socket;
127
+ try {
128
+ socket = this.options.createSocket(this.options.url);
129
+ }
130
+ catch {
131
+ this.scheduleReconnect();
132
+ return;
133
+ }
134
+ this.socket = socket;
135
+ socket.binaryType = "arraybuffer";
136
+ socket.onopen = () => this.sendText({ op: "hello", v: 1, peer: this.options.peerId });
137
+ socket.onmessage = (event) => this.handleMessage(socket, event.data);
138
+ socket.onclose = () => this.handleClose(socket);
139
+ socket.onerror = () => undefined;
140
+ this.handshakeTimer = setTimeout(() => {
141
+ this.handshakeTimer = undefined;
142
+ // Give up on this attempt as if the socket had closed; the close event of a socket we abandon is ignored.
143
+ this.handleClose(socket);
144
+ }, RELAY_HANDSHAKE_TIMEOUT_MS);
145
+ }
146
+ handleMessage(socket, data) {
147
+ if (socket !== this.socket) {
148
+ return;
149
+ }
150
+ if (typeof data === "string") {
151
+ this.handleText(data);
152
+ return;
153
+ }
154
+ const bytes = toBytes(data);
155
+ if (!bytes || !this.isReady) {
156
+ return;
157
+ }
158
+ const idLength = bytes.length > 0 ? bytes[0] : 0;
159
+ if (idLength === 0 || bytes.length < 1 + idLength) {
160
+ return;
161
+ }
162
+ // A malformed id decodes to replacement characters, which match no receiver, so it needs no separate check.
163
+ const peerId = this.decoder.decode(bytes.subarray(1, 1 + idLength));
164
+ this.options.onMedia(peerId, bytes.subarray(1 + idLength));
165
+ }
166
+ handleText(text) {
167
+ let message;
168
+ try {
169
+ message = JSON.parse(text);
170
+ }
171
+ catch {
172
+ return;
173
+ }
174
+ if (!message || typeof message !== "object" || message.op !== "ready" || this.isReady) {
175
+ return;
176
+ }
177
+ this.isReady = true;
178
+ this.attempt = 0;
179
+ const announced = message.maxMessageBytes;
180
+ if (typeof announced === "number" && Number.isInteger(announced) && announced >= MIN_NEGOTIATED_MESSAGE_BYTES) {
181
+ this.messageLimit = Math.min(announced, MAX_NEGOTIATED_MESSAGE_BYTES);
182
+ }
183
+ clearTimeout(this.handshakeTimer);
184
+ this.handshakeTimer = undefined;
185
+ // The server's wanted set is per socket, so it starts empty on every (re)connect.
186
+ this.sendText({ op: "want", peers: this.wanted });
187
+ }
188
+ handleClose(socket) {
189
+ if (socket !== this.socket) {
190
+ return;
191
+ }
192
+ this.isReady = false;
193
+ // The next server might be a different one (a rolling upgrade), so the limit is learned afresh.
194
+ this.messageLimit = MAX_MESSAGE_BYTES;
195
+ clearTimeout(this.handshakeTimer);
196
+ this.handshakeTimer = undefined;
197
+ this.detach();
198
+ this.scheduleReconnect();
199
+ }
200
+ /** Detaches and closes the current socket, if any, so its late events can't reach this client. */
201
+ detach() {
202
+ const socket = this.socket;
203
+ this.socket = undefined;
204
+ if (!socket) {
205
+ return;
206
+ }
207
+ socket.onopen = socket.onmessage = socket.onclose = socket.onerror = null;
208
+ try {
209
+ socket.close(1000, "closing");
210
+ }
211
+ catch {
212
+ // Already gone.
213
+ }
214
+ }
215
+ scheduleReconnect() {
216
+ const ceiling = Math.min(RELAY_BACKOFF_MAX_MS, RELAY_BACKOFF_BASE_MS * 2 ** Math.min(this.attempt, 10));
217
+ const delay = ceiling / 2 + (this.options.random() * ceiling) / 2;
218
+ this.attempt += 1;
219
+ this.reconnectTimer = setTimeout(() => {
220
+ this.reconnectTimer = undefined;
221
+ this.openSocket();
222
+ }, delay);
223
+ }
224
+ sendText(message) {
225
+ const socket = this.socket;
226
+ if (!socket || socket.readyState !== SOCKET_OPEN) {
227
+ return;
228
+ }
229
+ try {
230
+ socket.send(JSON.stringify(message));
231
+ }
232
+ catch {
233
+ // The close that follows a dead socket reconnects and re-sends.
234
+ }
235
+ }
236
+ }
@@ -0,0 +1,21 @@
1
+ import type { AudioContextLike, RelayEnv } from "./relayEnv.js";
2
+ /**
3
+ * One remote participant as seen through the relay: reassembles their fragments, decodes them, and exposes the result
4
+ * as an ordinary `MediaStream` (`stream`) so the UI plays a relayed peer exactly like a WebRTC one.
5
+ *
6
+ * The stream exists, with both tracks, from construction - silent and blank until media arrives - so the caller can
7
+ * attach it to an element straight away and never has to swap a stream in later. Audio goes through a
8
+ * `MediaStreamAudioDestinationNode` on the shared playback context; video is drawn to a canvas and captured.
9
+ */
10
+ export declare class RelayReceiver {
11
+ readonly stream: MediaStream;
12
+ private readonly audioFrames;
13
+ private readonly videoFrames;
14
+ private readonly audio;
15
+ private readonly video;
16
+ constructor(env: RelayEnv, playbackContext: AudioContextLike);
17
+ /** Handles one relayed payload from this sender (a media fragment). Anything malformed is ignored. */
18
+ handleMedia(payload: Uint8Array): void;
19
+ /** Releases the decoders and ends the stream's tracks. */
20
+ close(): void;
21
+ }
@@ -0,0 +1,59 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ import { AudioPlayer } from "./AudioPlayer.js";
6
+ import { decodeFragment, FrameReassembler, KIND_AUDIO } from "./frames.js";
7
+ import { VideoPlayer } from "./VideoPlayer.js";
8
+ /** The canvas is sized to the first decoded picture; until then it is this size. */
9
+ const INITIAL_CANVAS_WIDTH = 320;
10
+ const INITIAL_CANVAS_HEIGHT = 240;
11
+ /** The frame rate `captureStream()` samples the canvas at, matching what the sender produces. */
12
+ const CAPTURE_FPS = 15;
13
+ /**
14
+ * One remote participant as seen through the relay: reassembles their fragments, decodes them, and exposes the result
15
+ * as an ordinary `MediaStream` (`stream`) so the UI plays a relayed peer exactly like a WebRTC one.
16
+ *
17
+ * The stream exists, with both tracks, from construction - silent and blank until media arrives - so the caller can
18
+ * attach it to an element straight away and never has to swap a stream in later. Audio goes through a
19
+ * `MediaStreamAudioDestinationNode` on the shared playback context; video is drawn to a canvas and captured.
20
+ */
21
+ export class RelayReceiver {
22
+ constructor(env, playbackContext) {
23
+ this.audioFrames = new FrameReassembler();
24
+ this.videoFrames = new FrameReassembler();
25
+ const destination = playbackContext.createMediaStreamDestination();
26
+ this.audio = new AudioPlayer(env, playbackContext, destination);
27
+ const canvas = env.createCanvas();
28
+ canvas.width = INITIAL_CANVAS_WIDTH;
29
+ canvas.height = INITIAL_CANVAS_HEIGHT;
30
+ const context = canvas.getContext("2d");
31
+ this.video = context ? new VideoPlayer(env, canvas, context) : undefined;
32
+ this.stream = env.createMediaStream([...destination.stream.getAudioTracks(), ...canvas.captureStream(CAPTURE_FPS).getVideoTracks()]);
33
+ }
34
+ /** Handles one relayed payload from this sender (a media fragment). Anything malformed is ignored. */
35
+ handleMedia(payload) {
36
+ const fragment = decodeFragment(payload);
37
+ if (!fragment) {
38
+ return;
39
+ }
40
+ const frame = (fragment.kind === KIND_AUDIO ? this.audioFrames : this.videoFrames).push(fragment);
41
+ if (!frame) {
42
+ return;
43
+ }
44
+ if (frame.kind === KIND_AUDIO) {
45
+ this.audio.push(frame);
46
+ }
47
+ else {
48
+ this.video?.push(frame);
49
+ }
50
+ }
51
+ /** Releases the decoders and ends the stream's tracks. */
52
+ close() {
53
+ this.audio.close();
54
+ this.video?.close();
55
+ for (const track of this.stream.getTracks()) {
56
+ track.stop();
57
+ }
58
+ }
59
+ }
@@ -0,0 +1,34 @@
1
+ import type { RelayEnv } from "./relayEnv.js";
2
+ import type { PublishFrame } from "./senderCommon.js";
3
+ /** Numbers and fragments the frames of one sender. The counters belong to the transport rather than to a capture
4
+ * session, so stopping and restarting sending never resets `seq` - a receiver still holding the old value would
5
+ * otherwise treat the restarted stream as older than what it has already seen. */
6
+ export declare class FramePublisher {
7
+ private readonly send;
8
+ private readonly maxMessageBytes;
9
+ private readonly seqs;
10
+ /** `send` returns whether one message was accepted by the socket. */
11
+ /** `maxMessageBytes` is the largest message the socket may send right now (it changes when the server's `ready`
12
+ * announces a limit), so it is read for every frame rather than fixed at construction. */
13
+ constructor(send: (message: Uint8Array) => boolean, maxMessageBytes?: () => number);
14
+ /** Fragments and sends one frame, stopping at the first fragment the socket refuses (the rest could not be
15
+ * used without it). Returns whether the whole frame went out. */
16
+ publish: PublishFrame;
17
+ }
18
+ export interface RelaySenderOptions {
19
+ env: RelayEnv;
20
+ send: (message: Uint8Array) => boolean;
21
+ canSend: () => boolean;
22
+ /** The socket's current message limit; defaults to what an older server accepts. */
23
+ maxMessageBytes?: () => number;
24
+ }
25
+ /** The publishing half of the relay: the two capture paths plus the frame numbering they share. */
26
+ export declare class RelaySender {
27
+ private readonly audio;
28
+ private readonly video;
29
+ constructor(options: RelaySenderOptions);
30
+ /** Starts or stops encoding whatever tracks are set. Stopping releases the encoders and audio nodes. */
31
+ setActive(active: boolean): void;
32
+ /** `null` sends nothing for that kind. Safe before or after `setActive`. */
33
+ setTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
34
+ }
@@ -0,0 +1,54 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ import { AudioSender } from "./AudioSender.js";
6
+ import { DEFAULT_MESSAGE_BYTES, fragmentFrame, KIND_AUDIO, KIND_VIDEO } from "./frames.js";
7
+ import { VideoSender } from "./VideoSender.js";
8
+ const SEQ_MODULUS = 0x10000;
9
+ /** Numbers and fragments the frames of one sender. The counters belong to the transport rather than to a capture
10
+ * session, so stopping and restarting sending never resets `seq` - a receiver still holding the old value would
11
+ * otherwise treat the restarted stream as older than what it has already seen. */
12
+ export class FramePublisher {
13
+ /** `send` returns whether one message was accepted by the socket. */
14
+ /** `maxMessageBytes` is the largest message the socket may send right now (it changes when the server's `ready`
15
+ * announces a limit), so it is read for every frame rather than fixed at construction. */
16
+ constructor(send, maxMessageBytes = () => DEFAULT_MESSAGE_BYTES) {
17
+ this.send = send;
18
+ this.maxMessageBytes = maxMessageBytes;
19
+ this.seqs = { [KIND_AUDIO]: 0, [KIND_VIDEO]: 0 };
20
+ /** Fragments and sends one frame, stopping at the first fragment the socket refuses (the rest could not be
21
+ * used without it). Returns whether the whole frame went out. */
22
+ this.publish = (kind, keyFrame, data, timestampMs) => {
23
+ const seq = this.seqs[kind];
24
+ this.seqs[kind] = (seq + 1) % SEQ_MODULUS;
25
+ const fragments = fragmentFrame(kind, keyFrame, seq, timestampMs, data, this.maxMessageBytes());
26
+ if (fragments.length === 0) {
27
+ return false;
28
+ }
29
+ for (const fragment of fragments) {
30
+ if (!this.send(fragment)) {
31
+ return false;
32
+ }
33
+ }
34
+ return true;
35
+ };
36
+ }
37
+ }
38
+ /** The publishing half of the relay: the two capture paths plus the frame numbering they share. */
39
+ export class RelaySender {
40
+ constructor(options) {
41
+ const deps = { env: options.env, publish: new FramePublisher(options.send, options.maxMessageBytes).publish, canSend: options.canSend };
42
+ this.audio = new AudioSender(deps);
43
+ this.video = new VideoSender(deps);
44
+ }
45
+ /** Starts or stops encoding whatever tracks are set. Stopping releases the encoders and audio nodes. */
46
+ setActive(active) {
47
+ this.audio.setActive(active);
48
+ this.video.setActive(active);
49
+ }
50
+ /** `null` sends nothing for that kind. Safe before or after `setActive`. */
51
+ setTrack(kind, track) {
52
+ (kind === "audio" ? this.audio : this.video).setTrack(track);
53
+ }
54
+ }
@@ -0,0 +1,38 @@
1
+ import { type RelayEnv } from "./relayEnv.js";
2
+ export interface RelayTransport {
3
+ /** false when this browser lacks WebSocket, AudioEncoder/AudioDecoder/VideoEncoder/VideoDecoder, VideoFrame,
4
+ * AudioContext or canvas.captureStream - the mesh then never falls back to the relay. */
5
+ readonly supported: boolean;
6
+ /** Start receiving `peerId`'s media (opens/keeps the socket, adds to `want`). `onStream` gets a MediaStream that
7
+ * plays that peer (one audio + one video track). Called at most once per stream creation; calling receiveFrom
8
+ * again for the same peer is a no-op. */
9
+ receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void;
10
+ stopReceivingFrom(peerId: string): void;
11
+ /** Start/stop publishing the local tracks (encode + send). Only encodes while active; stopping releases the
12
+ * encoders and audio nodes. */
13
+ setSending(active: boolean): void;
14
+ /** Same semantics as `MeshConnectionManager.setLocalTrack`: null = send nothing for that kind. Safe to call
15
+ * before or after `setSending`. */
16
+ setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
17
+ /** Closes the socket, all coders, streams and timers. Idempotent. */
18
+ close(): void;
19
+ }
20
+ export interface RelayTransportOptions {
21
+ meetingUid: string;
22
+ /** This participant's peer id on the meeting's signaling channel - the id the server stamps on what it sends. */
23
+ peerId: string;
24
+ /** Overrides the socket URL (default: `relayUrl(meetingUid)`). */
25
+ url?: string;
26
+ /** Overrides every browser API the relay uses (default: `detectRelayEnv()`), so a test can drive it with fakes. */
27
+ env?: RelayEnv;
28
+ }
29
+ /** The relay WebSocket's URL for a meeting: this deployment's origin (the API origin when the UI is served from a
30
+ * different one, else the page's own), `http` becoming `ws` - the same derivation `pushUrl()` uses for `/push`.
31
+ * `undefined` when there is no origin at all (not running in a browser). */
32
+ export declare function relayUrl(meetingUid: string): string | undefined;
33
+ /**
34
+ * Creates the relay transport for one meeting. Returns an inert transport (`supported: false`) when the browser lacks
35
+ * something the relay needs, so the caller can simply check `supported` once and never fall back to it.
36
+ * No socket is opened until the first `receiveFrom()` or `setSending(true)`.
37
+ */
38
+ export declare function createRelayTransport(options: RelayTransportOptions): RelayTransport;
@@ -0,0 +1,142 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ /**
6
+ * The last-resort media tier of a call: when two participants' WebRTC connection cannot be made at all (typically a
7
+ * network that blocks UDP and TURN alike), their audio and video travel over a WebSocket to the server, which fans
8
+ * each participant's frames out to whoever asked for them. This module is the facade the mesh manager talks to; the
9
+ * pieces behind it are:
10
+ *
11
+ * - `RelayClient` - the socket: hello/ready/want, reconnection, dropping media it cannot send promptly.
12
+ * - `frames` - the fragment format inside each WebSocket message and the reassembly of it.
13
+ * - `RelaySender` (`AudioSender`, `VideoSender`) - capture and encode the local tracks (Opus and VP8 via WebCodecs).
14
+ * - `RelayReceiver` (`AudioPlayer`, `VideoPlayer`) - decode a remote participant back into a `MediaStream`.
15
+ *
16
+ * There is one transport (one socket) per meeting, however many peers use it, and the codecs are fixed - there is no
17
+ * negotiation, because both ends are this same code.
18
+ */
19
+ import { apiOrigin } from "@rapidmx/react-shared/util/api.js";
20
+ import { PLAYBACK_SAMPLE_RATE } from "./AudioPlayer.js";
21
+ import { resumeAudioContext } from "./audioResume.js";
22
+ import { RelayClient } from "./RelayClient.js";
23
+ import { detectRelayEnv } from "./relayEnv.js";
24
+ import { RelayReceiver } from "./RelayReceiver.js";
25
+ import { RelaySender } from "./RelaySender.js";
26
+ /** The relay WebSocket's URL for a meeting: this deployment's origin (the API origin when the UI is served from a
27
+ * different one, else the page's own), `http` becoming `ws` - the same derivation `pushUrl()` uses for `/push`.
28
+ * `undefined` when there is no origin at all (not running in a browser). */
29
+ export function relayUrl(meetingUid) {
30
+ const origin = apiOrigin() || (typeof window !== "undefined" ? window.location.origin : "");
31
+ return origin ? `${origin.replace(/^http/i, "ws")}/api/mail/video-meetings/relay/${encodeURIComponent(meetingUid)}` : undefined;
32
+ }
33
+ /** What `createRelayTransport()` returns on a browser that cannot run the relay: every call is a no-op. */
34
+ const UNSUPPORTED_TRANSPORT = {
35
+ supported: false,
36
+ receiveFrom: () => undefined,
37
+ stopReceivingFrom: () => undefined,
38
+ setSending: () => undefined,
39
+ setLocalTrack: () => undefined,
40
+ close: () => undefined,
41
+ };
42
+ class WebSocketRelayTransport {
43
+ constructor(env, url, peerId) {
44
+ this.env = env;
45
+ this.supported = true;
46
+ this.receivers = new Map();
47
+ this.closed = false;
48
+ this.client = new RelayClient({
49
+ url,
50
+ peerId,
51
+ createSocket: (target) => env.createSocket(target),
52
+ random: () => env.random(),
53
+ onMedia: (sender, payload) => this.receivers.get(sender)?.handleMedia(payload),
54
+ });
55
+ this.sender = new RelaySender({
56
+ env,
57
+ send: (message) => this.client.sendMedia(message),
58
+ canSend: () => this.client.ready,
59
+ maxMessageBytes: () => this.client.maxMessageBytes,
60
+ });
61
+ }
62
+ receiveFrom(peerId, onStream) {
63
+ if (this.closed || this.receivers.has(peerId)) {
64
+ return;
65
+ }
66
+ const receiver = this.createReceiver();
67
+ if (!receiver) {
68
+ return;
69
+ }
70
+ this.receivers.set(peerId, receiver);
71
+ this.client.start();
72
+ this.client.setWanted(this.receivers.keys());
73
+ onStream(receiver.stream);
74
+ }
75
+ stopReceivingFrom(peerId) {
76
+ const receiver = this.receivers.get(peerId);
77
+ if (!receiver) {
78
+ return;
79
+ }
80
+ this.receivers.delete(peerId);
81
+ receiver.close();
82
+ this.client.setWanted(this.receivers.keys());
83
+ }
84
+ setSending(active) {
85
+ if (this.closed) {
86
+ return;
87
+ }
88
+ if (active) {
89
+ this.client.start();
90
+ }
91
+ this.sender.setActive(active);
92
+ }
93
+ setLocalTrack(kind, track) {
94
+ if (!this.closed) {
95
+ this.sender.setTrack(kind, track);
96
+ }
97
+ }
98
+ close() {
99
+ if (this.closed) {
100
+ return;
101
+ }
102
+ this.closed = true;
103
+ this.sender.setActive(false);
104
+ for (const receiver of this.receivers.values()) {
105
+ receiver.close();
106
+ }
107
+ this.receivers.clear();
108
+ this.client.close();
109
+ this.stopResume?.();
110
+ // `close()` rejects for a context that is already closed; nothing to do about it.
111
+ this.playback?.close().catch(() => undefined);
112
+ this.playback = undefined;
113
+ }
114
+ /** A receiver on the shared playback context, or `undefined` if this browser will not build one (the peer then
115
+ * simply has no relayed media). */
116
+ createReceiver() {
117
+ try {
118
+ if (!this.playback) {
119
+ this.playback = this.env.createAudioContext(PLAYBACK_SAMPLE_RATE);
120
+ // Nothing the user just did created this context, so it may start suspended - see `resumeAudioContext`.
121
+ this.stopResume = resumeAudioContext(this.playback, this.env.document);
122
+ }
123
+ return new RelayReceiver(this.env, this.playback);
124
+ }
125
+ catch {
126
+ return undefined;
127
+ }
128
+ }
129
+ }
130
+ /**
131
+ * Creates the relay transport for one meeting. Returns an inert transport (`supported: false`) when the browser lacks
132
+ * something the relay needs, so the caller can simply check `supported` once and never fall back to it.
133
+ * No socket is opened until the first `receiveFrom()` or `setSending(true)`.
134
+ */
135
+ export function createRelayTransport(options) {
136
+ const env = options.env ?? detectRelayEnv();
137
+ const url = options.url ?? relayUrl(options.meetingUid);
138
+ if (!env || !url) {
139
+ return UNSUPPORTED_TRANSPORT;
140
+ }
141
+ return new WebSocketRelayTransport(env, url, options.peerId);
142
+ }