@rapidmx/meet-plugin 0.6.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/apps/meet/[token].tsx +11 -10
- package/apps/meet/_CallControls.tsx +28 -2
- package/apps/meet/_CallView.tsx +26 -2
- package/apps/meet/_EffectsPanel.tsx +134 -0
- package/apps/meet/_MeetChrome.tsx +1 -1
- package/apps/meet/_MeetLobby.tsx +36 -4
- package/apps/meet/_ParticipantTile.tsx +73 -10
- package/apps/meet/_icons.tsx +4 -0
- package/apps/meet/_layout.tsx +2 -2
- package/apps/meet/_meetApi.ts +7 -1
- package/apps/meet/index.tsx +2 -2
- package/apps/settings-video-conferencing/_PersonalRoomCard.tsx +4 -4
- package/apps/settings-video-conferencing/_layout.tsx +2 -2
- package/apps/settings-video-conferencing/index.tsx +3 -3
- package/apps/shared/media/filters/VideoFilterProcessor.ts +408 -0
- package/apps/shared/media/filters/accessories.ts +262 -0
- package/apps/shared/media/filters/backgroundImage.ts +69 -0
- package/apps/shared/media/filters/filterTypes.ts +89 -0
- package/apps/shared/media/filters/mlModels.ts +174 -0
- package/apps/shared/media/filters/pixelEffects.ts +61 -0
- package/apps/shared/media/mediaPreferences.ts +105 -0
- package/apps/shared/media/useLocalMedia.ts +198 -14
- package/apps/shared/push/GuestSignalingClient.ts +4 -4
- package/apps/shared/relay/AudioPlayer.ts +111 -0
- package/apps/shared/relay/AudioSender.ts +194 -0
- package/apps/shared/relay/RelayClient.ts +262 -0
- package/apps/shared/relay/RelayReceiver.ts +72 -0
- package/apps/shared/relay/RelaySender.ts +74 -0
- package/apps/shared/relay/RelayTransport.ts +188 -0
- package/apps/shared/relay/VideoPlayer.ts +110 -0
- package/apps/shared/relay/VideoSender.ts +216 -0
- package/apps/shared/relay/audioResume.ts +55 -0
- package/apps/shared/relay/frames.ts +275 -0
- package/apps/shared/relay/relayEnv.ts +230 -0
- package/apps/shared/relay/senderCommon.ts +30 -0
- package/apps/shared/webrtc/MeshConnectionManager.ts +159 -7
- package/apps/shared/webrtc/realPeerConnection.ts +22 -0
- package/apps/shared/webrtc/types.ts +32 -2
- package/dist/apps/meet/[token].d.ts +3 -3
- package/dist/apps/meet/[token].js +10 -10
- package/dist/apps/meet/_CallControls.d.ts +2 -1
- package/dist/apps/meet/_CallControls.js +6 -3
- package/dist/apps/meet/_CallView.d.ts +4 -1
- package/dist/apps/meet/_CallView.js +20 -3
- package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
- package/dist/apps/meet/_EffectsPanel.js +60 -0
- package/dist/apps/meet/_MeetChrome.d.ts +1 -1
- package/dist/apps/meet/_MeetLobby.d.ts +8 -0
- package/dist/apps/meet/_MeetLobby.js +15 -4
- package/dist/apps/meet/_ParticipantTile.d.ts +17 -2
- package/dist/apps/meet/_ParticipantTile.js +27 -4
- package/dist/apps/meet/_icons.d.ts +1 -0
- package/dist/apps/meet/_icons.js +1 -0
- package/dist/apps/meet/_layout.d.ts +1 -1
- package/dist/apps/meet/_layout.js +1 -1
- package/dist/apps/meet/_meetApi.d.ts +6 -0
- package/dist/apps/meet/_meetApi.js +1 -1
- package/dist/apps/meet/index.js +2 -2
- package/dist/apps/settings-video-conferencing/_PersonalRoomCard.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/_PersonalRoomCard.js +4 -4
- package/dist/apps/settings-video-conferencing/_layout.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/_layout.js +1 -1
- package/dist/apps/settings-video-conferencing/index.d.ts +1 -1
- package/dist/apps/settings-video-conferencing/index.js +3 -3
- package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +78 -0
- package/dist/apps/shared/media/filters/VideoFilterProcessor.js +329 -0
- package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
- package/dist/apps/shared/media/filters/accessories.js +211 -0
- package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
- package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
- package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
- package/dist/apps/shared/media/filters/filterTypes.js +63 -0
- package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
- package/dist/apps/shared/media/filters/mlModels.js +106 -0
- package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
- package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
- package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
- package/dist/apps/shared/media/mediaPreferences.js +93 -0
- package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
- package/dist/apps/shared/media/useLocalMedia.js +160 -14
- package/dist/apps/shared/push/GuestSignalingClient.d.ts +1 -1
- package/dist/apps/shared/push/GuestSignalingClient.js +3 -3
- package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
- package/dist/apps/shared/relay/AudioPlayer.js +103 -0
- package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
- package/dist/apps/shared/relay/AudioSender.js +172 -0
- package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
- package/dist/apps/shared/relay/RelayClient.js +236 -0
- package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
- package/dist/apps/shared/relay/RelayReceiver.js +59 -0
- package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
- package/dist/apps/shared/relay/RelaySender.js +54 -0
- package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
- package/dist/apps/shared/relay/RelayTransport.js +142 -0
- package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
- package/dist/apps/shared/relay/VideoPlayer.js +102 -0
- package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
- package/dist/apps/shared/relay/VideoSender.js +204 -0
- package/dist/apps/shared/relay/audioResume.d.ts +12 -0
- package/dist/apps/shared/relay/audioResume.js +45 -0
- package/dist/apps/shared/relay/frames.d.ts +101 -0
- package/dist/apps/shared/relay/frames.js +221 -0
- package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
- package/dist/apps/shared/relay/relayEnv.js +69 -0
- package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
- package/dist/apps/shared/relay/senderCommon.js +10 -0
- package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +52 -2
- package/dist/apps/shared/webrtc/MeshConnectionManager.js +138 -7
- package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
- package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
- package/dist/apps/shared/webrtc/types.d.ts +29 -2
- package/dist/lib/index.js +2 -0
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
- package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
- package/dist/lib/util/EffectsUtils.js +21 -0
- package/dist/lib/util/EffectsUtils.js.map +1 -0
- package/dist/lib/util/IceServerUtils.js +25 -4
- package/dist/lib/util/IceServerUtils.js.map +1 -1
- package/dist/lib/util/RedisRelayBus.js +241 -0
- package/dist/lib/util/RedisRelayBus.js.map +1 -0
- package/dist/lib/util/RelayHub.js +405 -0
- package/dist/lib/util/RelayHub.js.map +1 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
- package/dist/types/util/EffectsUtils.d.ts +10 -0
- package/dist/types/util/IceServerUtils.d.ts +12 -1
- package/dist/types/util/RedisRelayBus.d.ts +131 -0
- package/dist/types/util/RelayHub.d.ts +188 -0
- package/package.json +25 -11
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
import { resumeAudioContext } from "./audioResume.js";
|
|
6
|
+
import { KIND_AUDIO } from "./frames.js";
|
|
7
|
+
import type {
|
|
8
|
+
AudioContextLike,
|
|
9
|
+
AudioDataLike,
|
|
10
|
+
AudioNodeLike,
|
|
11
|
+
EncodedChunkLike,
|
|
12
|
+
EncoderLike,
|
|
13
|
+
GainLike,
|
|
14
|
+
ScriptProcessorLike,
|
|
15
|
+
} from "./relayEnv.js";
|
|
16
|
+
import { safeClose, type SenderDeps } from "./senderCommon.js";
|
|
17
|
+
|
|
18
|
+
export const AUDIO_SAMPLE_RATE = 48_000;
|
|
19
|
+
/** Frames per `ScriptProcessorNode` callback: about 43 ms, a compromise between latency and how often the main
|
|
20
|
+
* thread is interrupted. */
|
|
21
|
+
export const AUDIO_PROCESS_FRAMES = 2048;
|
|
22
|
+
export const AUDIO_BITRATE = 24_000;
|
|
23
|
+
/** The Opus frame duration in microseconds (20 ms). */
|
|
24
|
+
export const AUDIO_FRAME_DURATION_US = 20_000;
|
|
25
|
+
/** Skip a block instead of queueing it when the encoder is this far behind. */
|
|
26
|
+
export const AUDIO_MAX_ENCODE_QUEUE = 8;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Captures one microphone track and publishes it as Opus.
|
|
30
|
+
*
|
|
31
|
+
* There is exactly one capture path, chosen for being testable and available everywhere the relay is: an
|
|
32
|
+
* `AudioContext` at 48 kHz, a `MediaStreamAudioSourceNode` on the track, and a `ScriptProcessorNode` that receives
|
|
33
|
+
* the raw PCM. (`ScriptProcessorNode` is deprecated in favour of `AudioWorklet`, but it needs no separate module
|
|
34
|
+
* file to load, which a plugin bundle has no good place for, and every browser still supports it.) The processor's
|
|
35
|
+
* output is routed through a zero-gain node to the destination, because browsers only run a processor that is
|
|
36
|
+
* connected to the destination, and the gain stops the microphone playing back locally.
|
|
37
|
+
*
|
|
38
|
+
* It encodes only while sending is active, a track is set and the socket can take the frames, and rebuilds the graph
|
|
39
|
+
* when the track is replaced. The encoder is created lazily and discarded on any error, so a coder that died is
|
|
40
|
+
* replaced at the next block rather than being repaired inside its own error callback.
|
|
41
|
+
*/
|
|
42
|
+
export class AudioSender {
|
|
43
|
+
private active = false;
|
|
44
|
+
private track: MediaStreamTrack | null = null;
|
|
45
|
+
private attachedTrack: MediaStreamTrack | null = null;
|
|
46
|
+
|
|
47
|
+
private ctx: AudioContextLike | undefined;
|
|
48
|
+
private source: AudioNodeLike | undefined;
|
|
49
|
+
private processor: ScriptProcessorLike | undefined;
|
|
50
|
+
private gain: GainLike | undefined;
|
|
51
|
+
private stopResume: (() => void) | undefined;
|
|
52
|
+
private encoder: EncoderLike<AudioDataLike> | undefined;
|
|
53
|
+
private baseMs = 0;
|
|
54
|
+
private samples = 0;
|
|
55
|
+
|
|
56
|
+
constructor(private readonly deps: SenderDeps) {}
|
|
57
|
+
|
|
58
|
+
setActive(active: boolean): void {
|
|
59
|
+
this.active = active;
|
|
60
|
+
this.sync();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
setTrack(track: MediaStreamTrack | null): void {
|
|
64
|
+
this.track = track;
|
|
65
|
+
this.sync();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Brings the capture graph in line with (`active`, `track`): torn down when either is missing, rebuilt when
|
|
69
|
+
* the track changed, untouched otherwise. */
|
|
70
|
+
private sync(): void {
|
|
71
|
+
const wanted = this.active ? this.track : null;
|
|
72
|
+
if (wanted === this.attachedTrack) {
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
this.detach();
|
|
76
|
+
if (wanted) {
|
|
77
|
+
this.attach(wanted);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private attach(track: MediaStreamTrack): void {
|
|
82
|
+
const env = this.deps.env;
|
|
83
|
+
this.attachedTrack = track;
|
|
84
|
+
this.baseMs = env.now();
|
|
85
|
+
this.samples = 0;
|
|
86
|
+
try {
|
|
87
|
+
const ctx = env.createAudioContext(AUDIO_SAMPLE_RATE);
|
|
88
|
+
this.ctx = ctx;
|
|
89
|
+
const source = ctx.createMediaStreamSource(env.createMediaStream([track]));
|
|
90
|
+
this.source = source;
|
|
91
|
+
const processor = ctx.createScriptProcessor(AUDIO_PROCESS_FRAMES, 1, 1);
|
|
92
|
+
this.processor = processor;
|
|
93
|
+
const gain = ctx.createGain();
|
|
94
|
+
gain.gain.value = 0;
|
|
95
|
+
this.gain = gain;
|
|
96
|
+
processor.onaudioprocess = (event) => this.process(event.inputBuffer.getChannelData(0));
|
|
97
|
+
source.connect(processor);
|
|
98
|
+
processor.connect(gain);
|
|
99
|
+
gain.connect(ctx.destination);
|
|
100
|
+
this.stopResume = resumeAudioContext(ctx, env.document);
|
|
101
|
+
} catch {
|
|
102
|
+
// No usable AudioContext (or a browser that refuses one): audio simply isn't relayed.
|
|
103
|
+
this.detach();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private detach(): void {
|
|
108
|
+
this.attachedTrack = null;
|
|
109
|
+
this.stopResume?.();
|
|
110
|
+
this.stopResume = undefined;
|
|
111
|
+
if (this.processor) {
|
|
112
|
+
this.processor.onaudioprocess = null;
|
|
113
|
+
}
|
|
114
|
+
this.source?.disconnect();
|
|
115
|
+
this.processor?.disconnect();
|
|
116
|
+
this.gain?.disconnect();
|
|
117
|
+
this.source = this.processor = this.gain = undefined;
|
|
118
|
+
const ctx = this.ctx;
|
|
119
|
+
this.ctx = undefined;
|
|
120
|
+
// `close()` rejects for a context that is already closed; nothing to do about it.
|
|
121
|
+
ctx?.close().catch(() => undefined);
|
|
122
|
+
this.discardEncoder();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
private discardEncoder(): void {
|
|
126
|
+
const encoder = this.encoder;
|
|
127
|
+
this.encoder = undefined;
|
|
128
|
+
safeClose(encoder);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private process(input: Float32Array): void {
|
|
132
|
+
const samples = input.length;
|
|
133
|
+
const timestampUs = Math.round((this.samples * 1_000_000) / AUDIO_SAMPLE_RATE);
|
|
134
|
+
this.samples += samples;
|
|
135
|
+
if (!this.deps.canSend()) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const encoder = this.encoder ?? this.createEncoder();
|
|
139
|
+
if (!encoder || encoder.encodeQueueSize > AUDIO_MAX_ENCODE_QUEUE) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
let audio: AudioDataLike | undefined;
|
|
143
|
+
try {
|
|
144
|
+
// The browser reuses `input` for the next block, so the encoder gets its own copy.
|
|
145
|
+
audio = this.deps.env.createAudioData({
|
|
146
|
+
format: "f32-planar",
|
|
147
|
+
sampleRate: AUDIO_SAMPLE_RATE,
|
|
148
|
+
numberOfFrames: samples,
|
|
149
|
+
numberOfChannels: 1,
|
|
150
|
+
timestamp: timestampUs,
|
|
151
|
+
data: new Float32Array(input),
|
|
152
|
+
});
|
|
153
|
+
encoder.encode(audio);
|
|
154
|
+
} catch {
|
|
155
|
+
this.discardEncoder();
|
|
156
|
+
} finally {
|
|
157
|
+
audio?.close();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
private createEncoder(): EncoderLike<AudioDataLike> | undefined {
|
|
162
|
+
let encoder: EncoderLike<AudioDataLike> | undefined;
|
|
163
|
+
try {
|
|
164
|
+
encoder = this.deps.env.createAudioEncoder({
|
|
165
|
+
output: (chunk) => this.onChunk(chunk),
|
|
166
|
+
error: () => {
|
|
167
|
+
// Only the current encoder's failure matters; a stale one was already replaced.
|
|
168
|
+
if (this.encoder === encoder) {
|
|
169
|
+
this.discardEncoder();
|
|
170
|
+
}
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
encoder.configure({
|
|
174
|
+
codec: "opus",
|
|
175
|
+
sampleRate: AUDIO_SAMPLE_RATE,
|
|
176
|
+
numberOfChannels: 1,
|
|
177
|
+
bitrate: AUDIO_BITRATE,
|
|
178
|
+
opus: { frameDuration: AUDIO_FRAME_DURATION_US },
|
|
179
|
+
});
|
|
180
|
+
} catch {
|
|
181
|
+
safeClose(encoder);
|
|
182
|
+
return undefined;
|
|
183
|
+
}
|
|
184
|
+
this.encoder = encoder;
|
|
185
|
+
return encoder;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private onChunk(chunk: EncodedChunkLike): void {
|
|
189
|
+
const bytes = new Uint8Array(chunk.byteLength);
|
|
190
|
+
chunk.copyTo(bytes);
|
|
191
|
+
// Every Opus packet decodes on its own, so each is flagged as a key frame.
|
|
192
|
+
this.deps.publish(KIND_AUDIO, true, bytes, this.baseMs + chunk.timestamp / 1000);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* The WebSocket to the server's media relay - the transport under the last-resort tier of a call, used for a pair of
|
|
7
|
+
* participants whose WebRTC connection could not be made to work at all.
|
|
8
|
+
*
|
|
9
|
+
* ## Wire protocol (fixed by the server)
|
|
10
|
+
*
|
|
11
|
+
* - The first message is text JSON `{"op":"hello","v":1,"peer":"<peerId>"}`; the server answers `{"op":"ready","v":1}`,
|
|
12
|
+
* plus `"maxMessageBytes"` - the largest client-to-server message it accepts - on a current server (absent on an
|
|
13
|
+
* older one, which accepts `MAX_MESSAGE_BYTES`). Nothing else may be sent before `ready`.
|
|
14
|
+
* - `{"op":"want","peers":[...]}` REPLACES the set of senders this socket receives from (the server accepts at most
|
|
15
|
+
* `MAX_WANTED_PEERS`). It has to be re-sent after every reconnect, because the server's set dies with the socket.
|
|
16
|
+
* - A binary message from the client is an opaque payload of at most `maxMessageBytes`. The server forwards it to
|
|
17
|
+
* every socket that wants this sender as `[1 byte N][N bytes UTF-8 sender peer id][payload]`, stamping the sender
|
|
18
|
+
* itself so a peer cannot speak as another.
|
|
19
|
+
* - Binary messages may be dropped under backpressure and the socket may close at any moment, so this client
|
|
20
|
+
* reconnects with exponential backoff and treats every media send as best effort.
|
|
21
|
+
*
|
|
22
|
+
* Authentication is the `jwt` cookie already in place (see `GuestSignalingClient`), so this client sets no cookie.
|
|
23
|
+
*/
|
|
24
|
+
import { DEFAULT_MESSAGE_BYTES } from "./frames.js";
|
|
25
|
+
import type { RelaySocket } from "./relayEnv.js";
|
|
26
|
+
|
|
27
|
+
/** The largest binary message an older server accepts - it closes the socket on a larger one (uWS's default limit).
|
|
28
|
+
* Assumed until the server's `ready` announces a different `maxMessageBytes`. */
|
|
29
|
+
export const MAX_MESSAGE_BYTES = DEFAULT_MESSAGE_BYTES;
|
|
30
|
+
/** A `maxMessageBytes` below this is not believed (nothing useful fits), and one above
|
|
31
|
+
* `MAX_NEGOTIATED_MESSAGE_BYTES` is clamped to it. */
|
|
32
|
+
export const MIN_NEGOTIATED_MESSAGE_BYTES = 1024;
|
|
33
|
+
export const MAX_NEGOTIATED_MESSAGE_BYTES = 1024 * 1024;
|
|
34
|
+
/** The most senders the server lets one socket `want`. */
|
|
35
|
+
export const MAX_WANTED_PEERS = 32;
|
|
36
|
+
/** A media send is dropped, not queued, while more than this many bytes are still waiting to leave - the
|
|
37
|
+
* connection is slower than the media, and queueing would only add latency to the frames that are still fresh. */
|
|
38
|
+
export const MAX_BUFFERED_BYTES = 256 * 1024;
|
|
39
|
+
/** Reconnect delay: about a second after the first failure, doubling to a ceiling, jittered so a server restart
|
|
40
|
+
* doesn't bring every client back in the same instant. */
|
|
41
|
+
export const RELAY_BACKOFF_BASE_MS = 1_000;
|
|
42
|
+
export const RELAY_BACKOFF_MAX_MS = 15_000;
|
|
43
|
+
/** How long a socket may take to become `ready` before it is abandoned and retried (a server that accepts the
|
|
44
|
+
* upgrade but never answers the hello would otherwise hang the tier forever). */
|
|
45
|
+
export const RELAY_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
46
|
+
|
|
47
|
+
const SOCKET_OPEN = 1;
|
|
48
|
+
|
|
49
|
+
export interface RelayClientOptions {
|
|
50
|
+
url: string;
|
|
51
|
+
peerId: string;
|
|
52
|
+
createSocket: (url: string) => RelaySocket;
|
|
53
|
+
random: () => number;
|
|
54
|
+
/** Called with the sender's peer id (as stamped by the server) and the original payload. */
|
|
55
|
+
onMedia: (peerId: string, payload: Uint8Array) => void;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Views `data` as bytes when it is binary (an `ArrayBuffer` or a typed array), else `undefined`. Uses
|
|
59
|
+
* `Object.prototype.toString` rather than `instanceof` so it works for a buffer from another realm too. */
|
|
60
|
+
function toBytes(data: unknown): Uint8Array | undefined {
|
|
61
|
+
if (ArrayBuffer.isView(data)) {
|
|
62
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
63
|
+
}
|
|
64
|
+
if (Object.prototype.toString.call(data) === "[object ArrayBuffer]") {
|
|
65
|
+
return new Uint8Array(data as ArrayBuffer);
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export class RelayClient {
|
|
71
|
+
private socket: RelaySocket | undefined;
|
|
72
|
+
private closed = false;
|
|
73
|
+
private started = false;
|
|
74
|
+
private isReady = false;
|
|
75
|
+
private messageLimit = MAX_MESSAGE_BYTES;
|
|
76
|
+
private attempt = 0;
|
|
77
|
+
private wanted: string[] = [];
|
|
78
|
+
private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
|
79
|
+
private handshakeTimer: ReturnType<typeof setTimeout> | undefined;
|
|
80
|
+
private readonly decoder = new TextDecoder();
|
|
81
|
+
|
|
82
|
+
constructor(private readonly options: RelayClientOptions) {}
|
|
83
|
+
|
|
84
|
+
/** True once the server has answered the hello on the current socket, i.e. media may be sent. */
|
|
85
|
+
get ready(): boolean {
|
|
86
|
+
return this.isReady;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The largest message `sendMedia()` will send: what the server announced in `ready` (validated and clamped), or
|
|
90
|
+
* `MAX_MESSAGE_BYTES` before that, without one, and again after every disconnect until the next `ready`. The
|
|
91
|
+
* frame fragmenter sizes its fragments from this. */
|
|
92
|
+
get maxMessageBytes(): number {
|
|
93
|
+
return this.messageLimit;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Opens the socket (and keeps it open, reconnecting as needed) until `close()`. Idempotent. */
|
|
97
|
+
start(): void {
|
|
98
|
+
if (this.started || this.closed) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
this.started = true;
|
|
102
|
+
this.openSocket();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Replaces the set of senders to receive from. Sent now if the socket is ready, and again after each reconnect. */
|
|
106
|
+
setWanted(peers: Iterable<string>): void {
|
|
107
|
+
this.wanted = [...peers].slice(0, MAX_WANTED_PEERS);
|
|
108
|
+
if (this.isReady) {
|
|
109
|
+
this.sendText({ op: "want", peers: this.wanted });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Sends one media message. Returns whether it was handed to the socket - `false` (dropped) when the socket is
|
|
114
|
+
* not ready, is backed up, or the message is too large for the server to accept. */
|
|
115
|
+
sendMedia(bytes: Uint8Array): boolean {
|
|
116
|
+
const socket = this.socket;
|
|
117
|
+
if (
|
|
118
|
+
!socket ||
|
|
119
|
+
!this.isReady ||
|
|
120
|
+
socket.readyState !== SOCKET_OPEN ||
|
|
121
|
+
socket.bufferedAmount > MAX_BUFFERED_BYTES ||
|
|
122
|
+
bytes.length > this.messageLimit
|
|
123
|
+
) {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
socket.send(bytes);
|
|
128
|
+
return true;
|
|
129
|
+
} catch {
|
|
130
|
+
// The close that follows a dead socket reconnects.
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Closes the socket and stops reconnecting for good. Idempotent. */
|
|
136
|
+
close(): void {
|
|
137
|
+
if (this.closed) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
this.closed = true;
|
|
141
|
+
this.isReady = false;
|
|
142
|
+
this.messageLimit = MAX_MESSAGE_BYTES;
|
|
143
|
+
clearTimeout(this.reconnectTimer);
|
|
144
|
+
clearTimeout(this.handshakeTimer);
|
|
145
|
+
this.reconnectTimer = this.handshakeTimer = undefined;
|
|
146
|
+
this.detach();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private openSocket(): void {
|
|
150
|
+
let socket: RelaySocket;
|
|
151
|
+
try {
|
|
152
|
+
socket = this.options.createSocket(this.options.url);
|
|
153
|
+
} catch {
|
|
154
|
+
this.scheduleReconnect();
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
this.socket = socket;
|
|
158
|
+
socket.binaryType = "arraybuffer";
|
|
159
|
+
socket.onopen = () => this.sendText({ op: "hello", v: 1, peer: this.options.peerId });
|
|
160
|
+
socket.onmessage = (event) => this.handleMessage(socket, event.data);
|
|
161
|
+
socket.onclose = () => this.handleClose(socket);
|
|
162
|
+
socket.onerror = () => undefined;
|
|
163
|
+
this.handshakeTimer = setTimeout(() => {
|
|
164
|
+
this.handshakeTimer = undefined;
|
|
165
|
+
// Give up on this attempt as if the socket had closed; the close event of a socket we abandon is ignored.
|
|
166
|
+
this.handleClose(socket);
|
|
167
|
+
}, RELAY_HANDSHAKE_TIMEOUT_MS);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private handleMessage(socket: RelaySocket, data: unknown): void {
|
|
171
|
+
if (socket !== this.socket) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (typeof data === "string") {
|
|
175
|
+
this.handleText(data);
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
const bytes = toBytes(data);
|
|
179
|
+
if (!bytes || !this.isReady) {
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const idLength = bytes.length > 0 ? bytes[0] : 0;
|
|
183
|
+
if (idLength === 0 || bytes.length < 1 + idLength) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
// A malformed id decodes to replacement characters, which match no receiver, so it needs no separate check.
|
|
187
|
+
const peerId = this.decoder.decode(bytes.subarray(1, 1 + idLength));
|
|
188
|
+
this.options.onMedia(peerId, bytes.subarray(1 + idLength));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private handleText(text: string): void {
|
|
192
|
+
let message: unknown;
|
|
193
|
+
try {
|
|
194
|
+
message = JSON.parse(text);
|
|
195
|
+
} catch {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (!message || typeof message !== "object" || (message as { op?: unknown }).op !== "ready" || this.isReady) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
this.isReady = true;
|
|
202
|
+
this.attempt = 0;
|
|
203
|
+
const announced = (message as { maxMessageBytes?: unknown }).maxMessageBytes;
|
|
204
|
+
if (typeof announced === "number" && Number.isInteger(announced) && announced >= MIN_NEGOTIATED_MESSAGE_BYTES) {
|
|
205
|
+
this.messageLimit = Math.min(announced, MAX_NEGOTIATED_MESSAGE_BYTES);
|
|
206
|
+
}
|
|
207
|
+
clearTimeout(this.handshakeTimer);
|
|
208
|
+
this.handshakeTimer = undefined;
|
|
209
|
+
// The server's wanted set is per socket, so it starts empty on every (re)connect.
|
|
210
|
+
this.sendText({ op: "want", peers: this.wanted });
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private handleClose(socket: RelaySocket): void {
|
|
214
|
+
if (socket !== this.socket) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
this.isReady = false;
|
|
218
|
+
// The next server might be a different one (a rolling upgrade), so the limit is learned afresh.
|
|
219
|
+
this.messageLimit = MAX_MESSAGE_BYTES;
|
|
220
|
+
clearTimeout(this.handshakeTimer);
|
|
221
|
+
this.handshakeTimer = undefined;
|
|
222
|
+
this.detach();
|
|
223
|
+
this.scheduleReconnect();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Detaches and closes the current socket, if any, so its late events can't reach this client. */
|
|
227
|
+
private detach(): void {
|
|
228
|
+
const socket = this.socket;
|
|
229
|
+
this.socket = undefined;
|
|
230
|
+
if (!socket) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
socket.onopen = socket.onmessage = socket.onclose = socket.onerror = null;
|
|
234
|
+
try {
|
|
235
|
+
socket.close(1000, "closing");
|
|
236
|
+
} catch {
|
|
237
|
+
// Already gone.
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private scheduleReconnect(): void {
|
|
242
|
+
const ceiling = Math.min(RELAY_BACKOFF_MAX_MS, RELAY_BACKOFF_BASE_MS * 2 ** Math.min(this.attempt, 10));
|
|
243
|
+
const delay = ceiling / 2 + (this.options.random() * ceiling) / 2;
|
|
244
|
+
this.attempt += 1;
|
|
245
|
+
this.reconnectTimer = setTimeout(() => {
|
|
246
|
+
this.reconnectTimer = undefined;
|
|
247
|
+
this.openSocket();
|
|
248
|
+
}, delay);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
private sendText(message: object): void {
|
|
252
|
+
const socket = this.socket;
|
|
253
|
+
if (!socket || socket.readyState !== SOCKET_OPEN) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
try {
|
|
257
|
+
socket.send(JSON.stringify(message));
|
|
258
|
+
} catch {
|
|
259
|
+
// The close that follows a dead socket reconnects and re-sends.
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
import { AudioPlayer } from "./AudioPlayer.js";
|
|
6
|
+
import { decodeFragment, FrameReassembler, KIND_AUDIO } from "./frames.js";
|
|
7
|
+
import type { AudioContextLike, RelayEnv } from "./relayEnv.js";
|
|
8
|
+
import { VideoPlayer } from "./VideoPlayer.js";
|
|
9
|
+
|
|
10
|
+
/** The canvas is sized to the first decoded picture; until then it is this size. */
|
|
11
|
+
const INITIAL_CANVAS_WIDTH = 320;
|
|
12
|
+
const INITIAL_CANVAS_HEIGHT = 240;
|
|
13
|
+
/** The frame rate `captureStream()` samples the canvas at, matching what the sender produces. */
|
|
14
|
+
const CAPTURE_FPS = 15;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* One remote participant as seen through the relay: reassembles their fragments, decodes them, and exposes the result
|
|
18
|
+
* as an ordinary `MediaStream` (`stream`) so the UI plays a relayed peer exactly like a WebRTC one.
|
|
19
|
+
*
|
|
20
|
+
* The stream exists, with both tracks, from construction - silent and blank until media arrives - so the caller can
|
|
21
|
+
* attach it to an element straight away and never has to swap a stream in later. Audio goes through a
|
|
22
|
+
* `MediaStreamAudioDestinationNode` on the shared playback context; video is drawn to a canvas and captured.
|
|
23
|
+
*/
|
|
24
|
+
export class RelayReceiver {
|
|
25
|
+
readonly stream: MediaStream;
|
|
26
|
+
private readonly audioFrames = new FrameReassembler();
|
|
27
|
+
private readonly videoFrames = new FrameReassembler();
|
|
28
|
+
private readonly audio: AudioPlayer;
|
|
29
|
+
private readonly video: VideoPlayer | undefined;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
env: RelayEnv,
|
|
33
|
+
playbackContext: AudioContextLike,
|
|
34
|
+
) {
|
|
35
|
+
const destination = playbackContext.createMediaStreamDestination();
|
|
36
|
+
this.audio = new AudioPlayer(env, playbackContext, destination);
|
|
37
|
+
|
|
38
|
+
const canvas = env.createCanvas();
|
|
39
|
+
canvas.width = INITIAL_CANVAS_WIDTH;
|
|
40
|
+
canvas.height = INITIAL_CANVAS_HEIGHT;
|
|
41
|
+
const context = canvas.getContext("2d");
|
|
42
|
+
this.video = context ? new VideoPlayer(env, canvas, context) : undefined;
|
|
43
|
+
|
|
44
|
+
this.stream = env.createMediaStream([...destination.stream.getAudioTracks(), ...canvas.captureStream(CAPTURE_FPS).getVideoTracks()]);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Handles one relayed payload from this sender (a media fragment). Anything malformed is ignored. */
|
|
48
|
+
handleMedia(payload: Uint8Array): void {
|
|
49
|
+
const fragment = decodeFragment(payload);
|
|
50
|
+
if (!fragment) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const frame = (fragment.kind === KIND_AUDIO ? this.audioFrames : this.videoFrames).push(fragment);
|
|
54
|
+
if (!frame) {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (frame.kind === KIND_AUDIO) {
|
|
58
|
+
this.audio.push(frame);
|
|
59
|
+
} else {
|
|
60
|
+
this.video?.push(frame);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Releases the decoders and ends the stream's tracks. */
|
|
65
|
+
close(): void {
|
|
66
|
+
this.audio.close();
|
|
67
|
+
this.video?.close();
|
|
68
|
+
for (const track of this.stream.getTracks()) {
|
|
69
|
+
track.stop();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
import { AudioSender } from "./AudioSender.js";
|
|
6
|
+
import { DEFAULT_MESSAGE_BYTES, fragmentFrame, type FrameKind, KIND_AUDIO, KIND_VIDEO } from "./frames.js";
|
|
7
|
+
import type { RelayEnv } from "./relayEnv.js";
|
|
8
|
+
import type { PublishFrame } from "./senderCommon.js";
|
|
9
|
+
import { VideoSender } from "./VideoSender.js";
|
|
10
|
+
|
|
11
|
+
const SEQ_MODULUS = 0x10000;
|
|
12
|
+
|
|
13
|
+
/** Numbers and fragments the frames of one sender. The counters belong to the transport rather than to a capture
|
|
14
|
+
* session, so stopping and restarting sending never resets `seq` - a receiver still holding the old value would
|
|
15
|
+
* otherwise treat the restarted stream as older than what it has already seen. */
|
|
16
|
+
export class FramePublisher {
|
|
17
|
+
private readonly seqs: Record<FrameKind, number> = { [KIND_AUDIO]: 0, [KIND_VIDEO]: 0 };
|
|
18
|
+
|
|
19
|
+
/** `send` returns whether one message was accepted by the socket. */
|
|
20
|
+
/** `maxMessageBytes` is the largest message the socket may send right now (it changes when the server's `ready`
|
|
21
|
+
* announces a limit), so it is read for every frame rather than fixed at construction. */
|
|
22
|
+
constructor(
|
|
23
|
+
private readonly send: (message: Uint8Array) => boolean,
|
|
24
|
+
private readonly maxMessageBytes: () => number = () => DEFAULT_MESSAGE_BYTES,
|
|
25
|
+
) {}
|
|
26
|
+
|
|
27
|
+
/** Fragments and sends one frame, stopping at the first fragment the socket refuses (the rest could not be
|
|
28
|
+
* used without it). Returns whether the whole frame went out. */
|
|
29
|
+
publish: PublishFrame = (kind, keyFrame, data, timestampMs) => {
|
|
30
|
+
const seq = this.seqs[kind];
|
|
31
|
+
this.seqs[kind] = (seq + 1) % SEQ_MODULUS;
|
|
32
|
+
const fragments = fragmentFrame(kind, keyFrame, seq, timestampMs, data, this.maxMessageBytes());
|
|
33
|
+
if (fragments.length === 0) {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
for (const fragment of fragments) {
|
|
37
|
+
if (!this.send(fragment)) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return true;
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RelaySenderOptions {
|
|
46
|
+
env: RelayEnv;
|
|
47
|
+
send: (message: Uint8Array) => boolean;
|
|
48
|
+
canSend: () => boolean;
|
|
49
|
+
/** The socket's current message limit; defaults to what an older server accepts. */
|
|
50
|
+
maxMessageBytes?: () => number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The publishing half of the relay: the two capture paths plus the frame numbering they share. */
|
|
54
|
+
export class RelaySender {
|
|
55
|
+
private readonly audio: AudioSender;
|
|
56
|
+
private readonly video: VideoSender;
|
|
57
|
+
|
|
58
|
+
constructor(options: RelaySenderOptions) {
|
|
59
|
+
const deps = { env: options.env, publish: new FramePublisher(options.send, options.maxMessageBytes).publish, canSend: options.canSend };
|
|
60
|
+
this.audio = new AudioSender(deps);
|
|
61
|
+
this.video = new VideoSender(deps);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Starts or stops encoding whatever tracks are set. Stopping releases the encoders and audio nodes. */
|
|
65
|
+
setActive(active: boolean): void {
|
|
66
|
+
this.audio.setActive(active);
|
|
67
|
+
this.video.setActive(active);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** `null` sends nothing for that kind. Safe before or after `setActive`. */
|
|
71
|
+
setTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void {
|
|
72
|
+
(kind === "audio" ? this.audio : this.video).setTrack(track);
|
|
73
|
+
}
|
|
74
|
+
}
|