@rapidmx/meet-plugin 0.5.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +15 -2
  2. package/apps/meet/[token].tsx +3 -2
  3. package/apps/meet/_CallControls.tsx +28 -2
  4. package/apps/meet/_CallView.tsx +23 -2
  5. package/apps/meet/_EffectsPanel.tsx +134 -0
  6. package/apps/meet/_MeetLobby.tsx +22 -0
  7. package/apps/meet/_ParticipantTile.tsx +62 -9
  8. package/apps/meet/_icons.tsx +4 -0
  9. package/apps/meet/_meetApi.ts +6 -0
  10. package/apps/shared/media/filters/VideoFilterProcessor.ts +383 -0
  11. package/apps/shared/media/filters/accessories.ts +262 -0
  12. package/apps/shared/media/filters/backgroundImage.ts +69 -0
  13. package/apps/shared/media/filters/filterTypes.ts +89 -0
  14. package/apps/shared/media/filters/mlModels.ts +174 -0
  15. package/apps/shared/media/filters/pixelEffects.ts +61 -0
  16. package/apps/shared/media/mediaPreferences.ts +105 -0
  17. package/apps/shared/media/useLocalMedia.ts +198 -14
  18. package/apps/shared/relay/AudioPlayer.ts +111 -0
  19. package/apps/shared/relay/AudioSender.ts +194 -0
  20. package/apps/shared/relay/RelayClient.ts +262 -0
  21. package/apps/shared/relay/RelayReceiver.ts +72 -0
  22. package/apps/shared/relay/RelaySender.ts +74 -0
  23. package/apps/shared/relay/RelayTransport.ts +188 -0
  24. package/apps/shared/relay/VideoPlayer.ts +110 -0
  25. package/apps/shared/relay/VideoSender.ts +216 -0
  26. package/apps/shared/relay/audioResume.ts +55 -0
  27. package/apps/shared/relay/frames.ts +275 -0
  28. package/apps/shared/relay/relayEnv.ts +230 -0
  29. package/apps/shared/relay/senderCommon.ts +30 -0
  30. package/apps/shared/webrtc/MeshConnectionManager.ts +158 -6
  31. package/apps/shared/webrtc/realPeerConnection.ts +22 -0
  32. package/apps/shared/webrtc/types.ts +32 -2
  33. package/dist/apps/meet/[token].js +3 -3
  34. package/dist/apps/meet/_CallControls.d.ts +2 -1
  35. package/dist/apps/meet/_CallControls.js +6 -3
  36. package/dist/apps/meet/_CallView.d.ts +4 -1
  37. package/dist/apps/meet/_CallView.js +17 -3
  38. package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
  39. package/dist/apps/meet/_EffectsPanel.js +60 -0
  40. package/dist/apps/meet/_MeetLobby.d.ts +4 -0
  41. package/dist/apps/meet/_MeetLobby.js +8 -1
  42. package/dist/apps/meet/_ParticipantTile.d.ts +7 -1
  43. package/dist/apps/meet/_ParticipantTile.js +20 -3
  44. package/dist/apps/meet/_icons.d.ts +1 -0
  45. package/dist/apps/meet/_icons.js +1 -0
  46. package/dist/apps/meet/_meetApi.d.ts +6 -0
  47. package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +75 -0
  48. package/dist/apps/shared/media/filters/VideoFilterProcessor.js +304 -0
  49. package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
  50. package/dist/apps/shared/media/filters/accessories.js +211 -0
  51. package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
  52. package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
  53. package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
  54. package/dist/apps/shared/media/filters/filterTypes.js +63 -0
  55. package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
  56. package/dist/apps/shared/media/filters/mlModels.js +106 -0
  57. package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
  58. package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
  59. package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
  60. package/dist/apps/shared/media/mediaPreferences.js +93 -0
  61. package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
  62. package/dist/apps/shared/media/useLocalMedia.js +160 -14
  63. package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
  64. package/dist/apps/shared/relay/AudioPlayer.js +103 -0
  65. package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
  66. package/dist/apps/shared/relay/AudioSender.js +172 -0
  67. package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
  68. package/dist/apps/shared/relay/RelayClient.js +236 -0
  69. package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
  70. package/dist/apps/shared/relay/RelayReceiver.js +59 -0
  71. package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
  72. package/dist/apps/shared/relay/RelaySender.js +54 -0
  73. package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
  74. package/dist/apps/shared/relay/RelayTransport.js +142 -0
  75. package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
  76. package/dist/apps/shared/relay/VideoPlayer.js +102 -0
  77. package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
  78. package/dist/apps/shared/relay/VideoSender.js +204 -0
  79. package/dist/apps/shared/relay/audioResume.d.ts +12 -0
  80. package/dist/apps/shared/relay/audioResume.js +45 -0
  81. package/dist/apps/shared/relay/frames.d.ts +101 -0
  82. package/dist/apps/shared/relay/frames.js +221 -0
  83. package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
  84. package/dist/apps/shared/relay/relayEnv.js +69 -0
  85. package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
  86. package/dist/apps/shared/relay/senderCommon.js +10 -0
  87. package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +51 -1
  88. package/dist/apps/shared/webrtc/MeshConnectionManager.js +137 -6
  89. package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
  90. package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
  91. package/dist/apps/shared/webrtc/types.d.ts +29 -2
  92. package/dist/lib/index.js +2 -0
  93. package/dist/lib/index.js.map +1 -1
  94. package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
  95. package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
  96. package/dist/lib/util/EffectsUtils.js +21 -0
  97. package/dist/lib/util/EffectsUtils.js.map +1 -0
  98. package/dist/lib/util/IceServerUtils.js +25 -4
  99. package/dist/lib/util/IceServerUtils.js.map +1 -1
  100. package/dist/lib/util/RedisRelayBus.js +241 -0
  101. package/dist/lib/util/RedisRelayBus.js.map +1 -0
  102. package/dist/lib/util/RelayHub.js +405 -0
  103. package/dist/lib/util/RelayHub.js.map +1 -0
  104. package/dist/types/index.d.ts +2 -0
  105. package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
  106. package/dist/types/util/EffectsUtils.d.ts +10 -0
  107. package/dist/types/util/IceServerUtils.d.ts +12 -1
  108. package/dist/types/util/RedisRelayBus.d.ts +131 -0
  109. package/dist/types/util/RelayHub.d.ts +188 -0
  110. package/package.json +24 -7
@@ -0,0 +1,221 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ /**
6
+ * The media frame format carried inside a relay WebSocket message - see `RelayClient.ts` for the envelope the
7
+ * server wraps around it. The server treats the payload as opaque bytes, so everything in this module is a contract
8
+ * between two browsers running this same code, and it has to survive an unreliable transport: a WebSocket to the
9
+ * relay is ordered and lossless per connection, but the server drops binary messages under backpressure, so a
10
+ * receiver must cope with missing fragments without throwing or growing its memory without bound.
11
+ *
12
+ * ## Layout (all big-endian)
13
+ *
14
+ * A frame (one encoded audio packet or one encoded video picture) is split into fragments sized to the largest
15
+ * WebSocket message the server accepts (`fragmentPayloadSize()`): an older server allows only 16384 bytes (a larger
16
+ * message closes the socket) and a key frame easily runs to tens of kilobytes, while a current one announces a
17
+ * larger limit in its `ready` reply. Each fragment is
18
+ *
19
+ * `[kind u8][flags u8][seq u16][fragIndex u8][fragCount u8][timestampMs u32][fragment bytes]`
20
+ *
21
+ * `seq` is a per-kind frame counter that wraps at 65536, so a receiver can tell fragments of different frames
22
+ * apart and notice a gap. `timestampMs` is the sender's monotonic clock and also wraps (at about 49.7 days), which
23
+ * is why the receiver runs it through `TimestampUnwrapper` before handing it to a decoder.
24
+ */
25
+ export const KIND_AUDIO = 1;
26
+ export const KIND_VIDEO = 2;
27
+ /** Bit 0 of the flags byte: this frame is independently decodable (always set for audio packets). */
28
+ export const FLAG_KEY_FRAME = 1;
29
+ export const HEADER_BYTES = 10;
30
+ /** The largest client-to-server message an older server accepts, and what a client assumes until the server's
31
+ * `ready` says otherwise. */
32
+ export const DEFAULT_MESSAGE_BYTES = 16384;
33
+ /** The payload size used under the default limit. It is well below 16384 - 10 so an older server keeps behaving
34
+ * exactly as it always has, with headroom besides. */
35
+ export const DEFAULT_FRAGMENT_BYTES = 12000;
36
+ /** The ceiling on the payload of one fragment, however large a message the server accepts: bigger messages gain
37
+ * nothing for a 480x360 stream and make one lost message cost more. */
38
+ export const MAX_FRAGMENT_BYTES = 60000;
39
+ /** The most fragments a frame may have (`fragCount` is one byte). */
40
+ export const MAX_FRAGMENTS = 255;
41
+ /** The largest frame a receiver will buffer. A 480x360 VP8 key frame at the rates the sender uses is a small
42
+ * fraction of this; the cap exists so a hostile or buggy peer cannot make a receiver allocate without bound. */
43
+ export const MAX_FRAME_BYTES = 1024 * 1024;
44
+ const SEQ_MODULUS = 0x10000;
45
+ const HALF_SEQ = 0x8000;
46
+ /** How many consecutive fragments from an "older" sequence are ignored before the receiver decides the sender
47
+ * restarted its counter and resynchronises to it. */
48
+ const RESYNC_AFTER_STALE = 64;
49
+ /** The payload bytes per fragment for a server that accepts messages of `maxMessageBytes`. At or below the
50
+ * default limit that is `DEFAULT_FRAGMENT_BYTES` (or less, for a smaller limit); above it, everything the limit
51
+ * leaves after the header, capped at `MAX_FRAGMENT_BYTES`. A message is therefore never larger than the limit. */
52
+ export function fragmentPayloadSize(maxMessageBytes) {
53
+ const room = Math.max(1, maxMessageBytes - HEADER_BYTES);
54
+ return maxMessageBytes <= DEFAULT_MESSAGE_BYTES ? Math.min(DEFAULT_FRAGMENT_BYTES, room) : Math.min(MAX_FRAGMENT_BYTES, room);
55
+ }
56
+ /** Splits `data` into wire-ready fragments (header included), each at most `maxMessageBytes` long (default: what an
57
+ * older server accepts). An empty frame still produces one (header-only) fragment. Returns an empty array for a frame
58
+ * the receiver would refuse anyway - larger than `MAX_FRAME_BYTES`, or needing more than `MAX_FRAGMENTS` fragments at
59
+ * this message size - so it is not worth sending. `seq` and `timestampMs` are reduced modulo their field widths, so a
60
+ * caller can pass a free-running counter or clock. */
61
+ export function fragmentFrame(kind, keyFrame, seq, timestampMs, data, maxMessageBytes = DEFAULT_MESSAGE_BYTES) {
62
+ const payload = fragmentPayloadSize(maxMessageBytes);
63
+ const fragCount = Math.max(1, Math.ceil(data.length / payload));
64
+ if (data.length > MAX_FRAME_BYTES || fragCount > MAX_FRAGMENTS) {
65
+ return [];
66
+ }
67
+ const fragments = [];
68
+ for (let index = 0; index < fragCount; index++) {
69
+ const part = data.subarray(index * payload, (index + 1) * payload);
70
+ const out = new Uint8Array(HEADER_BYTES + part.length);
71
+ const view = new DataView(out.buffer);
72
+ view.setUint8(0, kind);
73
+ view.setUint8(1, keyFrame ? FLAG_KEY_FRAME : 0);
74
+ view.setUint16(2, seq % SEQ_MODULUS);
75
+ view.setUint8(4, index);
76
+ view.setUint8(5, fragCount);
77
+ view.setUint32(6, timestampMs >>> 0);
78
+ out.set(part, HEADER_BYTES);
79
+ fragments.push(out);
80
+ }
81
+ return fragments;
82
+ }
83
+ /** Parses one fragment, or returns `undefined` for anything that is not a well-formed one (too short, unknown
84
+ * kind, a zero fragment count, an index beyond the count). Never throws: the bytes come off the network. */
85
+ export function decodeFragment(bytes) {
86
+ if (bytes.length < HEADER_BYTES) {
87
+ return undefined;
88
+ }
89
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
90
+ const kind = view.getUint8(0);
91
+ if (kind !== KIND_AUDIO && kind !== KIND_VIDEO) {
92
+ return undefined;
93
+ }
94
+ const fragIndex = view.getUint8(4);
95
+ const fragCount = view.getUint8(5);
96
+ if (fragCount === 0 || fragIndex >= fragCount) {
97
+ return undefined;
98
+ }
99
+ return {
100
+ kind,
101
+ keyFrame: (view.getUint8(1) & FLAG_KEY_FRAME) !== 0,
102
+ seq: view.getUint16(2),
103
+ fragIndex,
104
+ fragCount,
105
+ timestampMs: view.getUint32(6),
106
+ data: bytes.subarray(HEADER_BYTES),
107
+ };
108
+ }
109
+ /** True when `a` is a later sequence number than `b`, treating the 16 bit counter as circular. */
110
+ export function seqIsNewer(a, b) {
111
+ const diff = (a - b + SEQ_MODULUS) % SEQ_MODULUS;
112
+ return diff !== 0 && diff < HALF_SEQ;
113
+ }
114
+ /**
115
+ * Collects the fragments of one sender's one media kind back into whole frames. It holds at most one frame in
116
+ * flight: the moment a fragment of a newer sequence arrives, an incomplete older frame is abandoned (its missing
117
+ * piece was dropped by the server, and waiting longer would only add latency). Everything is validated and nothing
118
+ * throws, so a garbage or hostile stream can at worst cost the frames it corrupts.
119
+ */
120
+ export class FrameReassembler {
121
+ constructor() {
122
+ this.stale = 0;
123
+ }
124
+ /** Feeds one fragment. Returns the frame if this fragment completed it. */
125
+ push(fragment) {
126
+ if (this.lastSeq === undefined || seqIsNewer(fragment.seq, this.lastSeq)) {
127
+ this.stale = 0;
128
+ this.lastSeq = fragment.seq;
129
+ this.pending = undefined;
130
+ }
131
+ else if (fragment.seq !== this.lastSeq) {
132
+ // Older than what is already being (or has been) collected: a straggler, unless the sender restarted its
133
+ // counter and everything it sends now looks older, in which case follow it after a run of them.
134
+ this.stale += 1;
135
+ if (this.stale < RESYNC_AFTER_STALE) {
136
+ return undefined;
137
+ }
138
+ this.stale = 0;
139
+ this.lastSeq = fragment.seq;
140
+ this.pending = undefined;
141
+ }
142
+ else if (!this.pending) {
143
+ // A further fragment of a frame that was already delivered, or that was abandoned as too big.
144
+ return undefined;
145
+ }
146
+ if (fragment.fragCount === 1 && !this.pending) {
147
+ // The common case (every audio packet, most delta frames): nothing to collect.
148
+ return this.finish(fragment, [fragment.data], fragment.data.length);
149
+ }
150
+ return this.collect(fragment);
151
+ }
152
+ /** Drops any frame in flight, e.g. after the consumer reset its decoder. */
153
+ reset() {
154
+ this.pending = undefined;
155
+ this.lastSeq = undefined;
156
+ this.stale = 0;
157
+ }
158
+ collect(fragment) {
159
+ let pending = this.pending;
160
+ if (!pending) {
161
+ pending = {
162
+ seq: fragment.seq,
163
+ fragCount: fragment.fragCount,
164
+ keyFrame: fragment.keyFrame,
165
+ timestampMs: fragment.timestampMs,
166
+ parts: new Array(fragment.fragCount).fill(undefined),
167
+ received: 0,
168
+ bytes: 0,
169
+ };
170
+ this.pending = pending;
171
+ }
172
+ if (fragment.fragCount !== pending.fragCount ||
173
+ fragment.keyFrame !== pending.keyFrame ||
174
+ fragment.timestampMs !== pending.timestampMs ||
175
+ pending.parts[fragment.fragIndex] !== undefined) {
176
+ // Inconsistent with the frame being built (or a duplicate): ignore this fragment, keep what is there.
177
+ return undefined;
178
+ }
179
+ pending.bytes += fragment.data.length;
180
+ if (pending.bytes > MAX_FRAME_BYTES) {
181
+ // Too big to be a frame this sender could legitimately have produced. Abandon it; `lastSeq` stays put so
182
+ // its remaining fragments are ignored rather than starting a fresh collection.
183
+ this.pending = undefined;
184
+ return undefined;
185
+ }
186
+ pending.parts[fragment.fragIndex] = fragment.data;
187
+ pending.received += 1;
188
+ if (pending.received < pending.fragCount) {
189
+ return undefined;
190
+ }
191
+ this.pending = undefined;
192
+ return this.finish(fragment, pending.parts, pending.bytes);
193
+ }
194
+ finish(fragment, parts, total) {
195
+ const data = new Uint8Array(total);
196
+ let offset = 0;
197
+ for (const part of parts) {
198
+ data.set(part, offset);
199
+ offset += part.length;
200
+ }
201
+ return { kind: fragment.kind, keyFrame: fragment.keyFrame, seq: fragment.seq, timestampMs: fragment.timestampMs, data };
202
+ }
203
+ }
204
+ /**
205
+ * Turns the wrapping 32 bit millisecond timestamps into a monotonic number a decoder can be given. The signed
206
+ * difference from the previous value is taken modulo 2^32, so it is correct across the wrap point and also for a
207
+ * frame that arrives slightly out of order (a small negative step).
208
+ */
209
+ export class TimestampUnwrapper {
210
+ constructor() {
211
+ this.unwrapped = 0;
212
+ }
213
+ /** Returns the unwrapped time in milliseconds relative to the first timestamp seen. */
214
+ next(timestampMs) {
215
+ if (this.last !== undefined) {
216
+ this.unwrapped += (timestampMs - this.last) | 0;
217
+ }
218
+ this.last = timestampMs;
219
+ return this.unwrapped;
220
+ }
221
+ }
@@ -0,0 +1,181 @@
1
+ /**
2
+ * Every browser API the relay tier touches, gathered behind one injectable object.
3
+ *
4
+ * WebCodecs, Web Audio and canvas capture are exactly the surfaces jsdom does not have, and the sender and
5
+ * receiver need a lot of them at once, so instead of patching a dozen globals in each test (and reading `globalThis`
6
+ * all over the implementation) the code takes a `RelayEnv` and `detectRelayEnv()` builds the real one. The interfaces
7
+ * here are deliberately the narrowest structural subset the relay actually calls - they are satisfied by the real
8
+ * browser objects, and small enough that a test's fake is a few lines.
9
+ */
10
+ /** The half of an encoded chunk (`EncodedAudioChunk`/`EncodedVideoChunk`) an encoder's output callback reads. */
11
+ export interface EncodedChunkLike {
12
+ readonly type: "key" | "delta";
13
+ /** Microseconds, as given to the encoder. */
14
+ readonly timestamp: number;
15
+ readonly byteLength: number;
16
+ copyTo(destination: Uint8Array): void;
17
+ }
18
+ export interface CoderInit<T> {
19
+ output: (output: T, metadata?: unknown) => void;
20
+ error: (error: Error) => void;
21
+ }
22
+ /** `AudioData`. */
23
+ export interface AudioDataLike {
24
+ readonly numberOfFrames: number;
25
+ readonly sampleRate: number;
26
+ copyTo(destination: Float32Array, options: {
27
+ planeIndex: number;
28
+ format?: string;
29
+ }): void;
30
+ close(): void;
31
+ }
32
+ export interface AudioDataInit {
33
+ format: "f32-planar";
34
+ sampleRate: number;
35
+ numberOfFrames: number;
36
+ numberOfChannels: number;
37
+ timestamp: number;
38
+ data: Float32Array;
39
+ }
40
+ /** `VideoFrame`. */
41
+ export interface VideoFrameLike {
42
+ readonly displayWidth: number;
43
+ readonly displayHeight: number;
44
+ close(): void;
45
+ }
46
+ export interface EncoderLike<TInput, TOptions = undefined> {
47
+ readonly encodeQueueSize: number;
48
+ configure(config: Record<string, unknown>): void;
49
+ encode(input: TInput, options?: TOptions): void;
50
+ close(): void;
51
+ }
52
+ export interface DecoderLike {
53
+ readonly decodeQueueSize: number;
54
+ configure(config: Record<string, unknown>): void;
55
+ decode(chunk: unknown): void;
56
+ close(): void;
57
+ }
58
+ export interface AudioNodeLike {
59
+ connect(destination: unknown): unknown;
60
+ disconnect(): void;
61
+ }
62
+ export interface ScriptProcessorLike extends AudioNodeLike {
63
+ onaudioprocess: ((event: {
64
+ inputBuffer: {
65
+ getChannelData(channel: number): Float32Array;
66
+ };
67
+ }) => void) | null;
68
+ }
69
+ export interface GainLike extends AudioNodeLike {
70
+ gain: {
71
+ value: number;
72
+ };
73
+ }
74
+ export interface AudioBufferLike {
75
+ copyToChannel(source: Float32Array, channel: number): void;
76
+ readonly duration: number;
77
+ }
78
+ export interface BufferSourceLike extends AudioNodeLike {
79
+ buffer: AudioBufferLike | null;
80
+ onended: (() => void) | null;
81
+ start(when?: number): void;
82
+ }
83
+ export interface AudioContextLike {
84
+ readonly state: string;
85
+ readonly currentTime: number;
86
+ readonly destination: unknown;
87
+ createMediaStreamSource(stream: MediaStream): AudioNodeLike;
88
+ createScriptProcessor(bufferSize: number, inputChannels: number, outputChannels: number): ScriptProcessorLike;
89
+ createGain(): GainLike;
90
+ createMediaStreamDestination(): {
91
+ readonly stream: MediaStream;
92
+ };
93
+ createBuffer(channels: number, length: number, sampleRate: number): AudioBufferLike;
94
+ createBufferSource(): BufferSourceLike;
95
+ resume(): Promise<void>;
96
+ close(): Promise<void>;
97
+ }
98
+ export interface CanvasContextLike {
99
+ drawImage(source: unknown, dx: number, dy: number, dw?: number, dh?: number): void;
100
+ }
101
+ /** A hidden `<canvas>`. */
102
+ export interface CanvasLike {
103
+ width: number;
104
+ height: number;
105
+ getContext(type: "2d"): CanvasContextLike | null;
106
+ captureStream(frameRate: number): MediaStream;
107
+ }
108
+ /** A hidden `<video>`. */
109
+ export interface VideoElementLike {
110
+ muted: boolean;
111
+ autoplay: boolean;
112
+ playsInline: boolean;
113
+ srcObject: unknown;
114
+ readonly readyState: number;
115
+ readonly videoWidth: number;
116
+ readonly videoHeight: number;
117
+ play(): Promise<void> | undefined;
118
+ pause(): void;
119
+ }
120
+ /** The subset of `WebSocket` the relay client uses (the same seam shape `GuestSignalingClient`'s `PushSocket` has). */
121
+ export interface RelaySocket {
122
+ readyState: number;
123
+ /** Bytes queued by `send()` that the browser has not yet handed to the network - the backpressure signal. */
124
+ readonly bufferedAmount: number;
125
+ binaryType: string;
126
+ send(data: string | Uint8Array): void;
127
+ close(code?: number, reason?: string): void;
128
+ onopen: ((event: unknown) => void) | null;
129
+ onmessage: ((event: {
130
+ data: unknown;
131
+ }) => void) | null;
132
+ onclose: ((event: unknown) => void) | null;
133
+ onerror: ((event: unknown) => void) | null;
134
+ }
135
+ /** What `resumeAudioContext()` needs from `document`. */
136
+ export interface EventTargetLike {
137
+ addEventListener(type: string, listener: () => void, options?: boolean): void;
138
+ removeEventListener(type: string, listener: () => void, options?: boolean): void;
139
+ }
140
+ export interface RelayEnv {
141
+ createSocket(url: string): RelaySocket;
142
+ /** `new AudioContext({ sampleRate })`. */
143
+ createAudioContext(sampleRate: number): AudioContextLike;
144
+ createMediaStream(tracks: MediaStreamTrack[]): MediaStream;
145
+ createVideoElement(): VideoElementLike;
146
+ createCanvas(): CanvasLike;
147
+ createAudioEncoder(init: CoderInit<EncodedChunkLike>): EncoderLike<AudioDataLike>;
148
+ createAudioDecoder(init: CoderInit<AudioDataLike>): DecoderLike;
149
+ createVideoEncoder(init: CoderInit<EncodedChunkLike>): EncoderLike<VideoFrameLike, {
150
+ keyFrame: boolean;
151
+ }>;
152
+ createVideoDecoder(init: CoderInit<VideoFrameLike>): DecoderLike;
153
+ createAudioData(init: AudioDataInit): AudioDataLike;
154
+ /** `new VideoFrame(canvas, { timestamp })`. */
155
+ createVideoFrame(canvas: CanvasLike, timestampUs: number): VideoFrameLike;
156
+ createEncodedAudioChunk(init: {
157
+ type: "key" | "delta";
158
+ timestamp: number;
159
+ data: Uint8Array;
160
+ }): unknown;
161
+ createEncodedVideoChunk(init: {
162
+ type: "key" | "delta";
163
+ timestamp: number;
164
+ data: Uint8Array;
165
+ }): unknown;
166
+ document: EventTargetLike;
167
+ setInterval(callback: () => void, ms: number): unknown;
168
+ clearInterval(handle: unknown): void;
169
+ /** A monotonic clock in milliseconds (`performance.now()`). */
170
+ now(): number;
171
+ random(): number;
172
+ }
173
+ /** The names of the globals `detectRelayEnv()` requires, in one place so the check and the tests agree. */
174
+ export declare const REQUIRED_GLOBALS: readonly ["WebSocket", "AudioEncoder", "AudioDecoder", "VideoEncoder", "VideoDecoder", "AudioData", "VideoFrame", "EncodedAudioChunk", "EncodedVideoChunk", "AudioContext", "MediaStream", "HTMLCanvasElement", "document"];
175
+ /**
176
+ * Builds the real environment from the browser's globals, or returns `undefined` when this browser lacks any
177
+ * piece of it (no WebSocket, no WebCodecs, no `AudioContext`, no `canvas.captureStream`, or not a browser at all).
178
+ * The relay codecs are fixed (Opus and VP8), so there is nothing useful to do with a partial environment - the
179
+ * mesh simply never falls back to the relay on such a browser.
180
+ */
181
+ export declare function detectRelayEnv(): RelayEnv | undefined;
@@ -0,0 +1,69 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ /**
6
+ * Every browser API the relay tier touches, gathered behind one injectable object.
7
+ *
8
+ * WebCodecs, Web Audio and canvas capture are exactly the surfaces jsdom does not have, and the sender and
9
+ * receiver need a lot of them at once, so instead of patching a dozen globals in each test (and reading `globalThis`
10
+ * all over the implementation) the code takes a `RelayEnv` and `detectRelayEnv()` builds the real one. The interfaces
11
+ * here are deliberately the narrowest structural subset the relay actually calls - they are satisfied by the real
12
+ * browser objects, and small enough that a test's fake is a few lines.
13
+ */
14
+ /** The names of the globals `detectRelayEnv()` requires, in one place so the check and the tests agree. */
15
+ export const REQUIRED_GLOBALS = [
16
+ "WebSocket",
17
+ "AudioEncoder",
18
+ "AudioDecoder",
19
+ "VideoEncoder",
20
+ "VideoDecoder",
21
+ "AudioData",
22
+ "VideoFrame",
23
+ "EncodedAudioChunk",
24
+ "EncodedVideoChunk",
25
+ "AudioContext",
26
+ "MediaStream",
27
+ "HTMLCanvasElement",
28
+ "document",
29
+ ];
30
+ /**
31
+ * Builds the real environment from the browser's globals, or returns `undefined` when this browser lacks any
32
+ * piece of it (no WebSocket, no WebCodecs, no `AudioContext`, no `canvas.captureStream`, or not a browser at all).
33
+ * The relay codecs are fixed (Opus and VP8), so there is nothing useful to do with a partial environment - the
34
+ * mesh simply never falls back to the relay on such a browser.
35
+ */
36
+ export function detectRelayEnv() {
37
+ const scope = globalThis;
38
+ for (const name of REQUIRED_GLOBALS) {
39
+ if (typeof scope[name] === "undefined") {
40
+ return undefined;
41
+ }
42
+ }
43
+ if (typeof HTMLCanvasElement.prototype.captureStream !== "function") {
44
+ return undefined;
45
+ }
46
+ // Each factory resolves its constructor when called (not once here) so the environment follows whatever the page
47
+ // has at that moment, and so a test can stub a constructor after detection.
48
+ const construct = (name, ...args) => new scope[name](...args);
49
+ return {
50
+ createSocket: (url) => construct("WebSocket", url),
51
+ createAudioContext: (sampleRate) => construct("AudioContext", { sampleRate }),
52
+ createMediaStream: (tracks) => construct("MediaStream", tracks),
53
+ createVideoElement: () => document.createElement("video"),
54
+ createCanvas: () => document.createElement("canvas"),
55
+ createAudioEncoder: (init) => construct("AudioEncoder", init),
56
+ createAudioDecoder: (init) => construct("AudioDecoder", init),
57
+ createVideoEncoder: (init) => construct("VideoEncoder", init),
58
+ createVideoDecoder: (init) => construct("VideoDecoder", init),
59
+ createAudioData: (init) => construct("AudioData", init),
60
+ createVideoFrame: (canvas, timestamp) => construct("VideoFrame", canvas, { timestamp }),
61
+ createEncodedAudioChunk: (init) => construct("EncodedAudioChunk", init),
62
+ createEncodedVideoChunk: (init) => construct("EncodedVideoChunk", init),
63
+ document,
64
+ setInterval: (callback, ms) => globalThis.setInterval(callback, ms),
65
+ clearInterval: (handle) => globalThis.clearInterval(handle),
66
+ now: () => performance.now(),
67
+ random: () => Math.random(),
68
+ };
69
+ }
@@ -0,0 +1,19 @@
1
+ import type { FrameKind } from "./frames.js";
2
+ import type { RelayEnv } from "./relayEnv.js";
3
+ /** Hands one encoded frame to the transport. `timestampMs` is on the `env.now()` clock. Returns whether every
4
+ * fragment of it was accepted by the socket - `false` means it was dropped (not ready, or backed up), which for
5
+ * video means the receiver will need a fresh key frame. */
6
+ export type PublishFrame = (kind: FrameKind, keyFrame: boolean, data: Uint8Array, timestampMs: number) => boolean;
7
+ /** What `AudioSender` and `VideoSender` are given. */
8
+ export interface SenderDeps {
9
+ env: RelayEnv;
10
+ publish: PublishFrame;
11
+ /** Whether a frame encoded now could actually be sent. Encoding while the socket is down or still handshaking
12
+ * would burn CPU on frames that are then dropped, so the senders idle instead. */
13
+ canSend: () => boolean;
14
+ }
15
+ /** Closes an encoder/decoder, ignoring the `InvalidStateError` a browser throws for one that is already closed
16
+ * (which is what a coder that reported an error is). */
17
+ export declare function safeClose(coder: {
18
+ close(): void;
19
+ } | undefined): void;
@@ -0,0 +1,10 @@
1
+ /** Closes an encoder/decoder, ignoring the `InvalidStateError` a browser throws for one that is already closed
2
+ * (which is what a coder that reported an error is). */
3
+ export function safeClose(coder) {
4
+ try {
5
+ coder?.close();
6
+ }
7
+ catch {
8
+ // Already closed.
9
+ }
10
+ }
@@ -57,10 +57,37 @@
57
57
  * `presenter-release`) once it observes the winning claim. See `handlePresenterClaim()`.
58
58
  *
59
59
  * Sharing a screen is `setLocalTrack("video", screenTrack)`; stopping is `setLocalTrack("video", cameraTrack)`.
60
+ *
61
+ * ## Three media paths, tried in order
62
+ *
63
+ * Each pair of participants reaches for the best path that works, and `MeshParticipant.transport` says which one it
64
+ * got:
65
+ *
66
+ * 1. **Direct peer-to-peer** (`"p2p"`). The `RTCPeerConnection` is given the STUN servers and the TURN server
67
+ * together, and ICE ranks a direct pair above a relayed one, so this is what a pair gets whenever it can.
68
+ * 2. **Relayed through the TURN server** (`"turn"`). The same connection, on the relay candidate ICE falls back to
69
+ * when the direct pairs fail - a participant behind a symmetric NAT or a firewall that blocks direct UDP. Which of
70
+ * the two ICE picked is read from the selected candidate pair once connected (`RTCPeerConnectionLike.connectionType()`).
71
+ * 3. **Proxied by this server over a WebSocket** (`"websocket"`, `apps/shared/relay/`). For a participant whose
72
+ * network lets nothing but ordinary HTTPS out, neither of the above ever connects. A pair whose connection has not
73
+ * come up within `connectTimeoutMs`, has failed, or has stayed `disconnected` for `disconnectedGraceMs` gives up
74
+ * on WebRTC and switches to it, telling the other side with a `relay-fallback` message so both switch even when
75
+ * only one of them noticed. Nothing is renegotiated - the pair simply stops using the `RTCPeerConnection` - and the
76
+ * relay's stream replaces the one WebRTC would have filled (a `remote-stream` event again). When the relay is
77
+ * unavailable (turned off by the operator, or a browser without WebCodecs) the pair is marked `"failed"` and stays
78
+ * in the roster, rather than the participant vanishing with no explanation as it used to.
79
+ *
80
+ * A pair never moves back up: once on the relay it stays there for the rest of the call.
60
81
  */
61
- import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type SignalingChannel } from "./types.js";
82
+ import { type MeshEvent, type MeshParticipant, type ParticipantState, type RTCPeerConnectionFactory, type RelayTransportLike, type SignalingChannel } from "./types.js";
62
83
  /** The lexicographically smaller uid is always the offerer for that pair - see this module's doc comment. */
63
84
  export declare function isOfferer(selfId: string, peerId: string): boolean;
85
+ /** How long a connection may stay in `connecting` before the pair gives up on WebRTC. ICE with an unreachable TURN
86
+ * server can take the better part of a minute to report `failed`; a person waiting on a call will not. It is still long
87
+ * enough for a slow TURN allocation and a few round trips of signaling. */
88
+ export declare const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
89
+ /** How long a connection may stay `disconnected` (ICE lost the path but may still recover) before giving up on it. */
90
+ export declare const DEFAULT_DISCONNECTED_GRACE_MS = 8000;
64
91
  type MediaKind = "audio" | "video";
65
92
  export interface MeshConnectionManagerOptions {
66
93
  /** The authenticated uid - what `from` must be on the wire (see `SignalMessage.from`). */
@@ -81,6 +108,13 @@ export interface MeshConnectionManagerOptions {
81
108
  /** Builds the (initially empty) stream one peer's incoming tracks are gathered into. Defaults to
82
109
  * `new MediaStream()`; a test supplies a fake. */
83
110
  createMediaStream?: () => MediaStream;
111
+ /** The WebSocket media relay, the last-resort path - see this module's doc comment. Leave unset when the
112
+ * operator has turned it off; without one a pair WebRTC cannot connect is marked `"failed"`. */
113
+ relay?: RelayTransportLike;
114
+ /** Defaults to `DEFAULT_CONNECT_TIMEOUT_MS`. */
115
+ connectTimeoutMs?: number;
116
+ /** Defaults to `DEFAULT_DISCONNECTED_GRACE_MS`. */
117
+ disconnectedGraceMs?: number;
84
118
  }
85
119
  export declare class MeshConnectionManager {
86
120
  private readonly options;
@@ -131,6 +165,22 @@ export declare class MeshConnectionManager {
131
165
  private updatePeer;
132
166
  private handleBye;
133
167
  private createPeer;
168
+ /** Reacts to the peer connection's state: notes which path a connected pair got, and gives up on WebRTC for a
169
+ * pair that failed or stays disconnected - see `fallBack()`. Ignores a pair that already left WebRTC. */
170
+ private handleConnectionState;
171
+ /** Records whether a connected pair is direct or relayed through TURN, from the candidate pair ICE selected. */
172
+ private detectTransport;
173
+ private setTransport;
174
+ private clearTimers;
175
+ /**
176
+ * Gives up on WebRTC for `peer` and moves the pair to the WebSocket relay (the third path - see this module's doc
177
+ * comment), or marks it `"failed"` when there is no relay to move to. `notify` tells the other side, which may not
178
+ * have noticed yet; a pair told by the other side does not tell it back.
179
+ */
180
+ private fallBack;
181
+ private handleRelayFallback;
182
+ /** The relay only encodes and uploads the local media while at least one pair is actually using it. */
183
+ private syncRelaySending;
134
184
  private initiateOffer;
135
185
  private handleOffer;
136
186
  private handleAnswer;