@rapidmx/meet-plugin 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/shared/media/deviceMedia.ts +22 -4
- package/apps/shared/media/useLocalMedia.ts +31 -7
- package/apps/shared/push/GuestSignalingClient.ts +17 -6
- package/apps/shared/relay/AudioPlayer.ts +30 -4
- package/apps/shared/relay/relayEnv.ts +8 -1
- package/dist/apps/shared/media/deviceMedia.d.ts +11 -2
- package/dist/apps/shared/media/deviceMedia.js +20 -4
- package/dist/apps/shared/media/useLocalMedia.js +29 -7
- package/dist/apps/shared/push/GuestSignalingClient.d.ts +10 -1
- package/dist/apps/shared/push/GuestSignalingClient.js +17 -6
- package/dist/apps/shared/relay/AudioPlayer.d.ts +14 -2
- package/dist/apps/shared/relay/AudioPlayer.js +31 -4
- package/dist/apps/shared/relay/relayEnv.d.ts +5 -0
- package/package.json +1 -1
|
@@ -50,10 +50,28 @@ const UNSUPPORTED_ERROR: MediaAccessError = {
|
|
|
50
50
|
message: "This browser (or this page's connection) doesn't support camera/microphone access. Try a modern browser over HTTPS.",
|
|
51
51
|
};
|
|
52
52
|
|
|
53
|
+
/** A "no such device" error naming exactly what was missing: just the microphone, just the camera, or - when
|
|
54
|
+
* both were asked for (or a caller doesn't say, e.g. `listDevices()`/`requestDisplayMedia()`'s own failures, which
|
|
55
|
+
* have no single "kind" to name) - both. Exported so `useLocalMedia.ts`'s `acquireBoth()` can build the same
|
|
56
|
+
* "both missing" message after finding out separately that neither device exists, rather than only being able to
|
|
57
|
+
* report whichever one it happened to ask about last. */
|
|
58
|
+
export function notFoundError(wantsAudio: boolean, wantsVideo: boolean): MediaAccessError {
|
|
59
|
+
const message =
|
|
60
|
+
wantsAudio && !wantsVideo
|
|
61
|
+
? "No microphone was found on this device."
|
|
62
|
+
: wantsVideo && !wantsAudio
|
|
63
|
+
? "No camera was found on this device."
|
|
64
|
+
: "No camera or microphone was found on this device.";
|
|
65
|
+
return { kind: "not-found", message };
|
|
66
|
+
}
|
|
67
|
+
|
|
53
68
|
/** Maps a `getUserMedia()`/`getDisplayMedia()`/`enumerateDevices()` rejection to a friendly, already-complete
|
|
54
69
|
* message - the handful of `DOMException` names a browser actually raises for these calls, matched by name rather
|
|
55
|
-
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`).
|
|
56
|
-
|
|
70
|
+
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`). `requested` is the
|
|
71
|
+
* constraints the failed call actually asked for, so a "not found" error can name just the camera or just the
|
|
72
|
+
* microphone rather than always blaming both - omitted (by `listDevices()`/`requestDisplayMedia()`, neither of
|
|
73
|
+
* which asks for one kind specifically) it's treated as having asked for both. */
|
|
74
|
+
export function classifyMediaError(err: unknown, requested?: MediaStreamConstraints): MediaAccessError {
|
|
57
75
|
const name: string = err instanceof Error ? err.name : "";
|
|
58
76
|
switch (name) {
|
|
59
77
|
case "NotAllowedError":
|
|
@@ -61,7 +79,7 @@ export function classifyMediaError(err: unknown): MediaAccessError {
|
|
|
61
79
|
return { kind: "permission-denied", message: "Camera/microphone access was denied. Allow access in your browser and try again." };
|
|
62
80
|
case "NotFoundError":
|
|
63
81
|
case "OverconstrainedError":
|
|
64
|
-
return
|
|
82
|
+
return notFoundError(requested ? !!requested.audio : true, requested ? !!requested.video : true);
|
|
65
83
|
default:
|
|
66
84
|
return { kind: "unknown", message: "Could not access your camera or microphone. Please try again." };
|
|
67
85
|
}
|
|
@@ -107,7 +125,7 @@ export async function requestUserMedia(
|
|
|
107
125
|
try {
|
|
108
126
|
return { ok: true, value: await devices.getUserMedia(constraints) };
|
|
109
127
|
} catch (err) {
|
|
110
|
-
return { ok: false, error: classifyMediaError(err) };
|
|
128
|
+
return { ok: false, error: classifyMediaError(err, constraints) };
|
|
111
129
|
}
|
|
112
130
|
}
|
|
113
131
|
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
defaultMediaDevices,
|
|
39
39
|
isMediaDevicesSupported,
|
|
40
40
|
listDevices,
|
|
41
|
+
notFoundError,
|
|
41
42
|
requestUserMedia,
|
|
42
43
|
} from "./deviceMedia.js";
|
|
43
44
|
import { startLevelMeter } from "./levelMeter.js";
|
|
@@ -115,6 +116,25 @@ function statusFor(error: MediaAccessError): TrackStatus {
|
|
|
115
116
|
}
|
|
116
117
|
}
|
|
117
118
|
|
|
119
|
+
/** After asking for the camera and the microphone separately (`acquireBoth()`'s fallback, one at a time rather
|
|
120
|
+
* than together), the one error to show for both attempts together - naming both devices when both are actually
|
|
121
|
+
* missing, rather than just whichever of the two happened to be asked about last (which would otherwise say "no
|
|
122
|
+
* camera" even when the microphone is *also* missing, or say nothing at all when only the microphone is missing,
|
|
123
|
+
* since the camera's own success would otherwise clear the error the microphone's failure had just set). `null`
|
|
124
|
+
* when both succeeded. */
|
|
125
|
+
function combineErrors(audio: MediaAccessError | null, video: MediaAccessError | null): MediaAccessError | null {
|
|
126
|
+
if (!audio || !video) {
|
|
127
|
+
return audio ?? video;
|
|
128
|
+
}
|
|
129
|
+
if (audio.kind === "not-found" && video.kind === "not-found") {
|
|
130
|
+
return notFoundError(true, true);
|
|
131
|
+
}
|
|
132
|
+
// Different reasons - unusual (e.g. permission denied for one, missing hardware for the other). Permission is
|
|
133
|
+
// the more actionable of the two, so it's what's shown; otherwise there's no single message that honestly
|
|
134
|
+
// describes both, so this just picks one.
|
|
135
|
+
return audio.kind === "permission-denied" ? audio : video;
|
|
136
|
+
}
|
|
137
|
+
|
|
118
138
|
export interface LocalMediaOptions {
|
|
119
139
|
/** Where the filters' models are hosted, when not on the default CDN - see `filters/mlModels.ts`. */
|
|
120
140
|
effectsAssetsUrl?: string;
|
|
@@ -245,7 +265,7 @@ export function useLocalMedia({ effectsAssetsUrl }: LocalMediaOptions = {}): Loc
|
|
|
245
265
|
|
|
246
266
|
/** Asks for one kind on its own - a specific device when `deviceId` is given, else the one used last time. */
|
|
247
267
|
const acquire = useCallback(
|
|
248
|
-
async (kind: MediaKind, deviceId?: string): Promise<
|
|
268
|
+
async (kind: MediaKind, deviceId?: string): Promise<MediaAccessError | null> => {
|
|
249
269
|
const base = kind === "audio" ? AUDIO_CONSTRAINTS : VIDEO_CONSTRAINTS;
|
|
250
270
|
const prefs = ensurePrefs();
|
|
251
271
|
const constraint: MediaTrackConstraints = deviceId
|
|
@@ -257,13 +277,13 @@ export function useLocalMedia({ effectsAssetsUrl }: LocalMediaOptions = {}): Loc
|
|
|
257
277
|
setError(result.error);
|
|
258
278
|
setStatus((prev) => ({ ...prev, [kind]: statusFor(result.error) }));
|
|
259
279
|
}
|
|
260
|
-
return
|
|
280
|
+
return result.error;
|
|
261
281
|
}
|
|
262
282
|
adoptStream(result.value);
|
|
263
283
|
if (mountedRef.current) {
|
|
264
284
|
setError(null);
|
|
265
285
|
}
|
|
266
|
-
return
|
|
286
|
+
return null;
|
|
267
287
|
},
|
|
268
288
|
[adoptStream, ensurePrefs],
|
|
269
289
|
);
|
|
@@ -292,9 +312,13 @@ export function useLocalMedia({ effectsAssetsUrl }: LocalMediaOptions = {}): Loc
|
|
|
292
312
|
}
|
|
293
313
|
} else {
|
|
294
314
|
// No camera, no microphone, or the two together couldn't be satisfied: ask for each on its own, so one
|
|
295
|
-
// missing device doesn't cost the participant the other
|
|
296
|
-
|
|
297
|
-
await acquire("
|
|
315
|
+
// missing device doesn't cost the participant the other - then combine what each found into one
|
|
316
|
+
// message (see `combineErrors()`) rather than showing whichever of the two ran last.
|
|
317
|
+
const audioError = await acquire("audio");
|
|
318
|
+
const videoError = await acquire("video");
|
|
319
|
+
if (mountedRef.current) {
|
|
320
|
+
setError(combineErrors(audioError, videoError));
|
|
321
|
+
}
|
|
298
322
|
}
|
|
299
323
|
}, [acquire, adoptStream, ensurePrefs]);
|
|
300
324
|
|
|
@@ -359,7 +383,7 @@ export function useLocalMedia({ effectsAssetsUrl }: LocalMediaOptions = {}): Loc
|
|
|
359
383
|
setTracks(tracksRef.current);
|
|
360
384
|
}
|
|
361
385
|
}
|
|
362
|
-
if (await acquire(kind, deviceId)) {
|
|
386
|
+
if ((await acquire(kind, deviceId)) === null) {
|
|
363
387
|
remember(kind === "audio" ? { microphoneId: deviceId } : { cameraId: deviceId });
|
|
364
388
|
}
|
|
365
389
|
await refreshDevices();
|
|
@@ -40,9 +40,11 @@
|
|
|
40
40
|
* request carries no `Authorization` header at all and instead relies on `fetch()`'s own default same-origin
|
|
41
41
|
* credentials mode, which already attaches the browser's real `jwt` cookie to a same-origin request with zero
|
|
42
42
|
* extra code - `JWTStrategy.getAuthToken()`'s cookie fallback authenticates it exactly as it would any other
|
|
43
|
-
* same-origin authenticated call.
|
|
44
|
-
*
|
|
45
|
-
*
|
|
43
|
+
* same-origin authenticated call. Either way `send()` also attaches the CSRF double-submit header
|
|
44
|
+
* (`withCsrfHeader()`) - required server-side only for the cookie case, but harmless to send for the bearer one
|
|
45
|
+
* too - see `send()`'s own doc comment. Fire-and-forget either way, matching every other push publish in this
|
|
46
|
+
* codebase - a failed send is logged, never thrown, since a caller (`MeshConnectionManager`) has no meaningful
|
|
47
|
+
* per-message retry of its own.
|
|
46
48
|
* - An incoming meeting message arrives wrapped `{ type: "MESSAGE", channel, data: <the posted body> }` - this
|
|
47
49
|
* client unwraps it and delivers `data` (expected to be a `SignalMessage`) to `onMessage()` listeners.
|
|
48
50
|
*
|
|
@@ -63,7 +65,7 @@
|
|
|
63
65
|
* absent, as described above - the browser's real cookie was always going to win that write anyway, so the
|
|
64
66
|
* correct behavior is to not fight it and let it authenticate normally instead.
|
|
65
67
|
*/
|
|
66
|
-
import { apiOrigin } from "@rapidmx/web-client/lib/util/api.js";
|
|
68
|
+
import { apiOrigin, withCsrfHeader } from "@rapidmx/web-client/lib/util/api.js";
|
|
67
69
|
import { pushUrl } from "@rapidmx/web-client/lib/mail/pushClient.js";
|
|
68
70
|
import type { SignalMessage, SignalingChannel } from "../webrtc/types.js";
|
|
69
71
|
|
|
@@ -174,7 +176,16 @@ export class GuestSignalingClient implements SignalingChannel {
|
|
|
174
176
|
|
|
175
177
|
/** Publishes over `POST /push/:id`, with `token` as a bearer credential when present - see this module's doc
|
|
176
178
|
* comment on why this, unlike subscribing, needs no cookie workaround at all, and on the omitted-`token` case
|
|
177
|
-
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead
|
|
179
|
+
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead - and,
|
|
180
|
+
* because that's a *cookie*-authenticated request, also needs the matching CSRF header
|
|
181
|
+
* (`@rapidmx/web-client`'s `withCsrfHeader()`, the same double-submit echo `apiFetch()` applies to every other
|
|
182
|
+
* mutating call this server answers): without it, `RouteUtils.checkCsrf()` refuses the request with 403
|
|
183
|
+
* `"This request is missing a valid CSRF token."` before it ever reaches `BasePushRoute.send()`'s own ACL
|
|
184
|
+
* check - every `hello`/offer/answer/ICE candidate/`bye` a participant on that path sends 403s and is
|
|
185
|
+
* silently dropped (`send()` is fire-and-forget), which looks exactly like "stuck connecting, then no audio or
|
|
186
|
+
* video" from everyone else's side. `withCsrfHeader()` is harmless to also add on the bearer-token (guest)
|
|
187
|
+
* branch - the server only enforces the check for `req.auth.source === "cookie"` - so it's applied
|
|
188
|
+
* unconditionally here rather than only for the cookie case.
|
|
178
189
|
* Fire-and-forget: a failure is swallowed (there is nothing a signaling message's own sender can usefully do
|
|
179
190
|
* about a dropped publish beyond what the mesh's own `hello` re-announcement/renegotiation already tolerates -
|
|
180
191
|
* see `MeshConnectionManager`'s doc comment on fire-and-forget delivery). */
|
|
@@ -190,7 +201,7 @@ export class GuestSignalingClient implements SignalingChannel {
|
|
|
190
201
|
}
|
|
191
202
|
void fetchImpl(`${origin}/push/${encodeURIComponent(this.options.channel)}`, {
|
|
192
203
|
method: "POST",
|
|
193
|
-
headers,
|
|
204
|
+
headers: withCsrfHeader(headers, "POST"),
|
|
194
205
|
body: JSON.stringify(message),
|
|
195
206
|
// Only a `bye` needs to survive the page unloading - every other message is sent while the page (and
|
|
196
207
|
// the rest of this call's signaling) is very much alive. `keepalive` is a real-world liability to set
|
|
@@ -16,6 +16,10 @@ export const JITTER_BUFFER_SECONDS = 0.08;
|
|
|
16
16
|
export const MAX_AHEAD_SECONDS = 0.4;
|
|
17
17
|
/** Skip a packet instead of queueing it when the decoder is this far behind. */
|
|
18
18
|
export const AUDIO_MAX_DECODE_QUEUE = 16;
|
|
19
|
+
/** The length of the fade-in `play()` gives a block that doesn't pick up exactly where the last one left off (the
|
|
20
|
+
* first block, or the one after a gap) - long enough to mask the click such a discontinuity would otherwise
|
|
21
|
+
* produce, short enough that it is never heard as an actual fade. */
|
|
22
|
+
export const DISCONTINUITY_FADE_SECONDS = 0.004;
|
|
19
23
|
|
|
20
24
|
/**
|
|
21
25
|
* Decodes one sender's Opus packets and schedules them for playback into a `MediaStreamAudioDestinationNode`.
|
|
@@ -23,8 +27,16 @@ export const AUDIO_MAX_DECODE_QUEUE = 16;
|
|
|
23
27
|
* Each decoded block is copied into an `AudioBuffer` and started on its own `AudioBufferSourceNode` at
|
|
24
28
|
* `max(currentTime + jitter, nextTime)`, where `nextTime` is when the previous block ends - so consecutive blocks
|
|
25
29
|
* join seamlessly, and after a gap in the stream playback restarts with the jitter buffer's worth of headroom.
|
|
26
|
-
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
27
|
-
*
|
|
30
|
+
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
31
|
+
* *in terms of content* - but the shared decoder (reused across packets so it keeps its normal state between them,
|
|
32
|
+
* not recreated per packet) is never told a packet was skipped, so its internal prediction state no longer matches
|
|
33
|
+
* what it decodes next. Audibly this can be a click right at that seam, not just silence where the lost packet
|
|
34
|
+
* would have been - the same is true of the very first block (nothing at all came before it) and of the block
|
|
35
|
+
* after a stall long enough to need the jitter buffer's full headroom again. `play()` gives exactly those blocks a
|
|
36
|
+
* few milliseconds' fade-in (`DISCONTINUITY_FADE_SECONDS`) through a `GainNode` to mask it; an ordinary block that
|
|
37
|
+
* picks up exactly where the last one ended gets none - fading *every* block would itself be audible, a faint
|
|
38
|
+
* tremolo at the packet rate. The decoder is created lazily and discarded on any error, and replaced by the next
|
|
39
|
+
* packet.
|
|
28
40
|
*/
|
|
29
41
|
export class AudioPlayer {
|
|
30
42
|
private decoder: DecoderLike | undefined;
|
|
@@ -97,9 +109,23 @@ export class AudioPlayer {
|
|
|
97
109
|
buffer.copyToChannel(pcm, 0);
|
|
98
110
|
const source = this.ctx.createBufferSource();
|
|
99
111
|
source.buffer = buffer;
|
|
100
|
-
source.connect(this.destination);
|
|
101
|
-
source.onended = () => source.disconnect();
|
|
102
112
|
const start = Math.max(now + JITTER_BUFFER_SECONDS, this.nextTime);
|
|
113
|
+
if (start > this.nextTime) {
|
|
114
|
+
// A gap, not a continuation - the decoder's state no longer matches what it's about to decode (see
|
|
115
|
+
// this class's doc comment), so fade this block in rather than let it start abruptly.
|
|
116
|
+
const gain = this.ctx.createGain();
|
|
117
|
+
gain.gain.setValueAtTime(0, start);
|
|
118
|
+
gain.gain.linearRampToValueAtTime(1, start + DISCONTINUITY_FADE_SECONDS);
|
|
119
|
+
source.connect(gain);
|
|
120
|
+
gain.connect(this.destination);
|
|
121
|
+
source.onended = () => {
|
|
122
|
+
source.disconnect();
|
|
123
|
+
gain.disconnect();
|
|
124
|
+
};
|
|
125
|
+
} else {
|
|
126
|
+
source.connect(this.destination);
|
|
127
|
+
source.onended = () => source.disconnect();
|
|
128
|
+
}
|
|
103
129
|
source.start(start);
|
|
104
130
|
this.nextTime = start + buffer.duration;
|
|
105
131
|
} catch {
|
|
@@ -74,7 +74,14 @@ export interface ScriptProcessorLike extends AudioNodeLike {
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
export interface GainLike extends AudioNodeLike {
|
|
77
|
-
gain: {
|
|
77
|
+
gain: {
|
|
78
|
+
value: number;
|
|
79
|
+
/** `AudioParam.setValueAtTime()` - used to schedule a fade (`AudioPlayer`'s discontinuity fade-in), not
|
|
80
|
+
* needed for `AudioSender`'s own static, unscheduled mute (a plain `.value = 0` assignment). */
|
|
81
|
+
setValueAtTime(value: number, startTime: number): unknown;
|
|
82
|
+
/** `AudioParam.linearRampToValueAtTime()`. */
|
|
83
|
+
linearRampToValueAtTime(value: number, endTime: number): unknown;
|
|
84
|
+
};
|
|
78
85
|
}
|
|
79
86
|
|
|
80
87
|
export interface AudioBufferLike {
|
|
@@ -36,10 +36,19 @@ export type MediaResult<T> = {
|
|
|
36
36
|
export declare function defaultMediaDevices(): MediaDevicesLike | undefined;
|
|
37
37
|
/** Whether `devices.getUserMedia` is actually callable - the one capability every caller here needs at minimum. */
|
|
38
38
|
export declare function isMediaDevicesSupported(devices?: MediaDevicesLike | undefined): boolean;
|
|
39
|
+
/** A "no such device" error naming exactly what was missing: just the microphone, just the camera, or - when
|
|
40
|
+
* both were asked for (or a caller doesn't say, e.g. `listDevices()`/`requestDisplayMedia()`'s own failures, which
|
|
41
|
+
* have no single "kind" to name) - both. Exported so `useLocalMedia.ts`'s `acquireBoth()` can build the same
|
|
42
|
+
* "both missing" message after finding out separately that neither device exists, rather than only being able to
|
|
43
|
+
* report whichever one it happened to ask about last. */
|
|
44
|
+
export declare function notFoundError(wantsAudio: boolean, wantsVideo: boolean): MediaAccessError;
|
|
39
45
|
/** Maps a `getUserMedia()`/`getDisplayMedia()`/`enumerateDevices()` rejection to a friendly, already-complete
|
|
40
46
|
* message - the handful of `DOMException` names a browser actually raises for these calls, matched by name rather
|
|
41
|
-
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`).
|
|
42
|
-
|
|
47
|
+
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`). `requested` is the
|
|
48
|
+
* constraints the failed call actually asked for, so a "not found" error can name just the camera or just the
|
|
49
|
+
* microphone rather than always blaming both - omitted (by `listDevices()`/`requestDisplayMedia()`, neither of
|
|
50
|
+
* which asks for one kind specifically) it's treated as having asked for both. */
|
|
51
|
+
export declare function classifyMediaError(err: unknown, requested?: MediaStreamConstraints): MediaAccessError;
|
|
43
52
|
export interface DeviceLists {
|
|
44
53
|
cameras: MediaDeviceInfo[];
|
|
45
54
|
microphones: MediaDeviceInfo[];
|
|
@@ -29,10 +29,26 @@ const UNSUPPORTED_ERROR = {
|
|
|
29
29
|
kind: "unsupported",
|
|
30
30
|
message: "This browser (or this page's connection) doesn't support camera/microphone access. Try a modern browser over HTTPS.",
|
|
31
31
|
};
|
|
32
|
+
/** A "no such device" error naming exactly what was missing: just the microphone, just the camera, or - when
|
|
33
|
+
* both were asked for (or a caller doesn't say, e.g. `listDevices()`/`requestDisplayMedia()`'s own failures, which
|
|
34
|
+
* have no single "kind" to name) - both. Exported so `useLocalMedia.ts`'s `acquireBoth()` can build the same
|
|
35
|
+
* "both missing" message after finding out separately that neither device exists, rather than only being able to
|
|
36
|
+
* report whichever one it happened to ask about last. */
|
|
37
|
+
export function notFoundError(wantsAudio, wantsVideo) {
|
|
38
|
+
const message = wantsAudio && !wantsVideo
|
|
39
|
+
? "No microphone was found on this device."
|
|
40
|
+
: wantsVideo && !wantsAudio
|
|
41
|
+
? "No camera was found on this device."
|
|
42
|
+
: "No camera or microphone was found on this device.";
|
|
43
|
+
return { kind: "not-found", message };
|
|
44
|
+
}
|
|
32
45
|
/** Maps a `getUserMedia()`/`getDisplayMedia()`/`enumerateDevices()` rejection to a friendly, already-complete
|
|
33
46
|
* message - the handful of `DOMException` names a browser actually raises for these calls, matched by name rather
|
|
34
|
-
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`).
|
|
35
|
-
|
|
47
|
+
* than `instanceof DOMException` (a test's fake rejection need not be a real `DOMException`). `requested` is the
|
|
48
|
+
* constraints the failed call actually asked for, so a "not found" error can name just the camera or just the
|
|
49
|
+
* microphone rather than always blaming both - omitted (by `listDevices()`/`requestDisplayMedia()`, neither of
|
|
50
|
+
* which asks for one kind specifically) it's treated as having asked for both. */
|
|
51
|
+
export function classifyMediaError(err, requested) {
|
|
36
52
|
const name = err instanceof Error ? err.name : "";
|
|
37
53
|
switch (name) {
|
|
38
54
|
case "NotAllowedError":
|
|
@@ -40,7 +56,7 @@ export function classifyMediaError(err) {
|
|
|
40
56
|
return { kind: "permission-denied", message: "Camera/microphone access was denied. Allow access in your browser and try again." };
|
|
41
57
|
case "NotFoundError":
|
|
42
58
|
case "OverconstrainedError":
|
|
43
|
-
return
|
|
59
|
+
return notFoundError(requested ? !!requested.audio : true, requested ? !!requested.video : true);
|
|
44
60
|
default:
|
|
45
61
|
return { kind: "unknown", message: "Could not access your camera or microphone. Please try again." };
|
|
46
62
|
}
|
|
@@ -78,7 +94,7 @@ export async function requestUserMedia(constraints, devices = defaultMediaDevice
|
|
|
78
94
|
return { ok: true, value: await devices.getUserMedia(constraints) };
|
|
79
95
|
}
|
|
80
96
|
catch (err) {
|
|
81
|
-
return { ok: false, error: classifyMediaError(err) };
|
|
97
|
+
return { ok: false, error: classifyMediaError(err, constraints) };
|
|
82
98
|
}
|
|
83
99
|
}
|
|
84
100
|
/** Requests a screen/window/tab share via the browser's own picker. Audio is never requested (this design shares
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
* Nothing here runs during SSR: browser APIs are only touched from effects and event handlers.
|
|
33
33
|
*/
|
|
34
34
|
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
|
35
|
-
import { defaultMediaDevices, isMediaDevicesSupported, listDevices, requestUserMedia, } from "./deviceMedia.js";
|
|
35
|
+
import { defaultMediaDevices, isMediaDevicesSupported, listDevices, notFoundError, requestUserMedia, } from "./deviceMedia.js";
|
|
36
36
|
import { startLevelMeter } from "./levelMeter.js";
|
|
37
37
|
import { loadPicture, prepareBackgroundImage } from "./filters/backgroundImage.js";
|
|
38
38
|
import { NO_FILTERS, filtersActive } from "./filters/filterTypes.js";
|
|
@@ -55,6 +55,24 @@ function statusFor(error) {
|
|
|
55
55
|
return "error";
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
|
+
/** After asking for the camera and the microphone separately (`acquireBoth()`'s fallback, one at a time rather
|
|
59
|
+
* than together), the one error to show for both attempts together - naming both devices when both are actually
|
|
60
|
+
* missing, rather than just whichever of the two happened to be asked about last (which would otherwise say "no
|
|
61
|
+
* camera" even when the microphone is *also* missing, or say nothing at all when only the microphone is missing,
|
|
62
|
+
* since the camera's own success would otherwise clear the error the microphone's failure had just set). `null`
|
|
63
|
+
* when both succeeded. */
|
|
64
|
+
function combineErrors(audio, video) {
|
|
65
|
+
if (!audio || !video) {
|
|
66
|
+
return audio ?? video;
|
|
67
|
+
}
|
|
68
|
+
if (audio.kind === "not-found" && video.kind === "not-found") {
|
|
69
|
+
return notFoundError(true, true);
|
|
70
|
+
}
|
|
71
|
+
// Different reasons - unusual (e.g. permission denied for one, missing hardware for the other). Permission is
|
|
72
|
+
// the more actionable of the two, so it's what's shown; otherwise there's no single message that honestly
|
|
73
|
+
// describes both, so this just picks one.
|
|
74
|
+
return audio.kind === "permission-denied" ? audio : video;
|
|
75
|
+
}
|
|
58
76
|
/** `base`, asking for `preferredId` (the device used last time) when there is one - as a preference, not a demand. */
|
|
59
77
|
function preferring(base, preferredId) {
|
|
60
78
|
return preferredId ? { ...base, deviceId: { ideal: preferredId } } : base;
|
|
@@ -172,13 +190,13 @@ export function useLocalMedia({ effectsAssetsUrl } = {}) {
|
|
|
172
190
|
setError(result.error);
|
|
173
191
|
setStatus((prev) => ({ ...prev, [kind]: statusFor(result.error) }));
|
|
174
192
|
}
|
|
175
|
-
return
|
|
193
|
+
return result.error;
|
|
176
194
|
}
|
|
177
195
|
adoptStream(result.value);
|
|
178
196
|
if (mountedRef.current) {
|
|
179
197
|
setError(null);
|
|
180
198
|
}
|
|
181
|
-
return
|
|
199
|
+
return null;
|
|
182
200
|
}, [adoptStream, ensurePrefs]);
|
|
183
201
|
/** Asks for the camera and the microphone in a single prompt. */
|
|
184
202
|
const acquireBoth = useCallback(async () => {
|
|
@@ -206,9 +224,13 @@ export function useLocalMedia({ effectsAssetsUrl } = {}) {
|
|
|
206
224
|
}
|
|
207
225
|
else {
|
|
208
226
|
// No camera, no microphone, or the two together couldn't be satisfied: ask for each on its own, so one
|
|
209
|
-
// missing device doesn't cost the participant the other
|
|
210
|
-
|
|
211
|
-
await acquire("
|
|
227
|
+
// missing device doesn't cost the participant the other - then combine what each found into one
|
|
228
|
+
// message (see `combineErrors()`) rather than showing whichever of the two ran last.
|
|
229
|
+
const audioError = await acquire("audio");
|
|
230
|
+
const videoError = await acquire("video");
|
|
231
|
+
if (mountedRef.current) {
|
|
232
|
+
setError(combineErrors(audioError, videoError));
|
|
233
|
+
}
|
|
212
234
|
}
|
|
213
235
|
}, [acquire, adoptStream, ensurePrefs]);
|
|
214
236
|
const requestAccess = useCallback(async (kind) => {
|
|
@@ -265,7 +287,7 @@ export function useLocalMedia({ effectsAssetsUrl } = {}) {
|
|
|
265
287
|
setTracks(tracksRef.current);
|
|
266
288
|
}
|
|
267
289
|
}
|
|
268
|
-
if (await acquire(kind, deviceId)) {
|
|
290
|
+
if ((await acquire(kind, deviceId)) === null) {
|
|
269
291
|
remember(kind === "audio" ? { microphoneId: deviceId } : { cameraId: deviceId });
|
|
270
292
|
}
|
|
271
293
|
await refreshDevices();
|
|
@@ -60,7 +60,16 @@ export declare class GuestSignalingClient implements SignalingChannel {
|
|
|
60
60
|
onMessage(handler: (message: SignalMessage) => void): () => void;
|
|
61
61
|
/** Publishes over `POST /push/:id`, with `token` as a bearer credential when present - see this module's doc
|
|
62
62
|
* comment on why this, unlike subscribing, needs no cookie workaround at all, and on the omitted-`token` case
|
|
63
|
-
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead
|
|
63
|
+
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead - and,
|
|
64
|
+
* because that's a *cookie*-authenticated request, also needs the matching CSRF header
|
|
65
|
+
* (`@rapidmx/web-client`'s `withCsrfHeader()`, the same double-submit echo `apiFetch()` applies to every other
|
|
66
|
+
* mutating call this server answers): without it, `RouteUtils.checkCsrf()` refuses the request with 403
|
|
67
|
+
* `"This request is missing a valid CSRF token."` before it ever reaches `BasePushRoute.send()`'s own ACL
|
|
68
|
+
* check - every `hello`/offer/answer/ICE candidate/`bye` a participant on that path sends 403s and is
|
|
69
|
+
* silently dropped (`send()` is fire-and-forget), which looks exactly like "stuck connecting, then no audio or
|
|
70
|
+
* video" from everyone else's side. `withCsrfHeader()` is harmless to also add on the bearer-token (guest)
|
|
71
|
+
* branch - the server only enforces the check for `req.auth.source === "cookie"` - so it's applied
|
|
72
|
+
* unconditionally here rather than only for the cookie case.
|
|
64
73
|
* Fire-and-forget: a failure is swallowed (there is nothing a signaling message's own sender can usefully do
|
|
65
74
|
* about a dropped publish beyond what the mesh's own `hello` re-announcement/renegotiation already tolerates -
|
|
66
75
|
* see `MeshConnectionManager`'s doc comment on fire-and-forget delivery). */
|
|
@@ -40,9 +40,11 @@
|
|
|
40
40
|
* request carries no `Authorization` header at all and instead relies on `fetch()`'s own default same-origin
|
|
41
41
|
* credentials mode, which already attaches the browser's real `jwt` cookie to a same-origin request with zero
|
|
42
42
|
* extra code - `JWTStrategy.getAuthToken()`'s cookie fallback authenticates it exactly as it would any other
|
|
43
|
-
* same-origin authenticated call.
|
|
44
|
-
*
|
|
45
|
-
*
|
|
43
|
+
* same-origin authenticated call. Either way `send()` also attaches the CSRF double-submit header
|
|
44
|
+
* (`withCsrfHeader()`) - required server-side only for the cookie case, but harmless to send for the bearer one
|
|
45
|
+
* too - see `send()`'s own doc comment. Fire-and-forget either way, matching every other push publish in this
|
|
46
|
+
* codebase - a failed send is logged, never thrown, since a caller (`MeshConnectionManager`) has no meaningful
|
|
47
|
+
* per-message retry of its own.
|
|
46
48
|
* - An incoming meeting message arrives wrapped `{ type: "MESSAGE", channel, data: <the posted body> }` - this
|
|
47
49
|
* client unwraps it and delivers `data` (expected to be a `SignalMessage`) to `onMessage()` listeners.
|
|
48
50
|
*
|
|
@@ -63,7 +65,7 @@
|
|
|
63
65
|
* absent, as described above - the browser's real cookie was always going to win that write anyway, so the
|
|
64
66
|
* correct behavior is to not fight it and let it authenticate normally instead.
|
|
65
67
|
*/
|
|
66
|
-
import { apiOrigin } from "@rapidmx/web-client/lib/util/api.js";
|
|
68
|
+
import { apiOrigin, withCsrfHeader } from "@rapidmx/web-client/lib/util/api.js";
|
|
67
69
|
import { pushUrl } from "@rapidmx/web-client/lib/mail/pushClient.js";
|
|
68
70
|
const SOCKET_OPEN = 1;
|
|
69
71
|
function defaultSocketFactory() {
|
|
@@ -132,7 +134,16 @@ export class GuestSignalingClient {
|
|
|
132
134
|
}
|
|
133
135
|
/** Publishes over `POST /push/:id`, with `token` as a bearer credential when present - see this module's doc
|
|
134
136
|
* comment on why this, unlike subscribing, needs no cookie workaround at all, and on the omitted-`token` case
|
|
135
|
-
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead
|
|
137
|
+
* (an already-authenticated real caller), which relies on the browser's own real `jwt` cookie instead - and,
|
|
138
|
+
* because that's a *cookie*-authenticated request, also needs the matching CSRF header
|
|
139
|
+
* (`@rapidmx/web-client`'s `withCsrfHeader()`, the same double-submit echo `apiFetch()` applies to every other
|
|
140
|
+
* mutating call this server answers): without it, `RouteUtils.checkCsrf()` refuses the request with 403
|
|
141
|
+
* `"This request is missing a valid CSRF token."` before it ever reaches `BasePushRoute.send()`'s own ACL
|
|
142
|
+
* check - every `hello`/offer/answer/ICE candidate/`bye` a participant on that path sends 403s and is
|
|
143
|
+
* silently dropped (`send()` is fire-and-forget), which looks exactly like "stuck connecting, then no audio or
|
|
144
|
+
* video" from everyone else's side. `withCsrfHeader()` is harmless to also add on the bearer-token (guest)
|
|
145
|
+
* branch - the server only enforces the check for `req.auth.source === "cookie"` - so it's applied
|
|
146
|
+
* unconditionally here rather than only for the cookie case.
|
|
136
147
|
* Fire-and-forget: a failure is swallowed (there is nothing a signaling message's own sender can usefully do
|
|
137
148
|
* about a dropped publish beyond what the mesh's own `hello` re-announcement/renegotiation already tolerates -
|
|
138
149
|
* see `MeshConnectionManager`'s doc comment on fire-and-forget delivery). */
|
|
@@ -148,7 +159,7 @@ export class GuestSignalingClient {
|
|
|
148
159
|
}
|
|
149
160
|
void fetchImpl(`${origin}/push/${encodeURIComponent(this.options.channel)}`, {
|
|
150
161
|
method: "POST",
|
|
151
|
-
headers,
|
|
162
|
+
headers: withCsrfHeader(headers, "POST"),
|
|
152
163
|
body: JSON.stringify(message),
|
|
153
164
|
// Only a `bye` needs to survive the page unloading - every other message is sent while the page (and
|
|
154
165
|
// the rest of this call's signaling) is very much alive. `keepalive` is a real-world liability to set
|
|
@@ -10,14 +10,26 @@ export declare const JITTER_BUFFER_SECONDS = 0.08;
|
|
|
10
10
|
export declare const MAX_AHEAD_SECONDS = 0.4;
|
|
11
11
|
/** Skip a packet instead of queueing it when the decoder is this far behind. */
|
|
12
12
|
export declare const AUDIO_MAX_DECODE_QUEUE = 16;
|
|
13
|
+
/** The length of the fade-in `play()` gives a block that doesn't pick up exactly where the last one left off (the
|
|
14
|
+
* first block, or the one after a gap) - long enough to mask the click such a discontinuity would otherwise
|
|
15
|
+
* produce, short enough that it is never heard as an actual fade. */
|
|
16
|
+
export declare const DISCONTINUITY_FADE_SECONDS = 0.004;
|
|
13
17
|
/**
|
|
14
18
|
* Decodes one sender's Opus packets and schedules them for playback into a `MediaStreamAudioDestinationNode`.
|
|
15
19
|
*
|
|
16
20
|
* Each decoded block is copied into an `AudioBuffer` and started on its own `AudioBufferSourceNode` at
|
|
17
21
|
* `max(currentTime + jitter, nextTime)`, where `nextTime` is when the previous block ends - so consecutive blocks
|
|
18
22
|
* join seamlessly, and after a gap in the stream playback restarts with the jitter buffer's worth of headroom.
|
|
19
|
-
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
20
|
-
*
|
|
23
|
+
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
24
|
+
* *in terms of content* - but the shared decoder (reused across packets so it keeps its normal state between them,
|
|
25
|
+
* not recreated per packet) is never told a packet was skipped, so its internal prediction state no longer matches
|
|
26
|
+
* what it decodes next. Audibly this can be a click right at that seam, not just silence where the lost packet
|
|
27
|
+
* would have been - the same is true of the very first block (nothing at all came before it) and of the block
|
|
28
|
+
* after a stall long enough to need the jitter buffer's full headroom again. `play()` gives exactly those blocks a
|
|
29
|
+
* few milliseconds' fade-in (`DISCONTINUITY_FADE_SECONDS`) through a `GainNode` to mask it; an ordinary block that
|
|
30
|
+
* picks up exactly where the last one ended gets none - fading *every* block would itself be audible, a faint
|
|
31
|
+
* tremolo at the packet rate. The decoder is created lazily and discarded on any error, and replaced by the next
|
|
32
|
+
* packet.
|
|
21
33
|
*/
|
|
22
34
|
export declare class AudioPlayer {
|
|
23
35
|
private readonly env;
|
|
@@ -14,14 +14,26 @@ export const JITTER_BUFFER_SECONDS = 0.08;
|
|
|
14
14
|
export const MAX_AHEAD_SECONDS = 0.4;
|
|
15
15
|
/** Skip a packet instead of queueing it when the decoder is this far behind. */
|
|
16
16
|
export const AUDIO_MAX_DECODE_QUEUE = 16;
|
|
17
|
+
/** The length of the fade-in `play()` gives a block that doesn't pick up exactly where the last one left off (the
|
|
18
|
+
* first block, or the one after a gap) - long enough to mask the click such a discontinuity would otherwise
|
|
19
|
+
* produce, short enough that it is never heard as an actual fade. */
|
|
20
|
+
export const DISCONTINUITY_FADE_SECONDS = 0.004;
|
|
17
21
|
/**
|
|
18
22
|
* Decodes one sender's Opus packets and schedules them for playback into a `MediaStreamAudioDestinationNode`.
|
|
19
23
|
*
|
|
20
24
|
* Each decoded block is copied into an `AudioBuffer` and started on its own `AudioBufferSourceNode` at
|
|
21
25
|
* `max(currentTime + jitter, nextTime)`, where `nextTime` is when the previous block ends - so consecutive blocks
|
|
22
26
|
* join seamlessly, and after a gap in the stream playback restarts with the jitter buffer's worth of headroom.
|
|
23
|
-
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
24
|
-
*
|
|
27
|
+
* Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step
|
|
28
|
+
* *in terms of content* - but the shared decoder (reused across packets so it keeps its normal state between them,
|
|
29
|
+
* not recreated per packet) is never told a packet was skipped, so its internal prediction state no longer matches
|
|
30
|
+
* what it decodes next. Audibly this can be a click right at that seam, not just silence where the lost packet
|
|
31
|
+
* would have been - the same is true of the very first block (nothing at all came before it) and of the block
|
|
32
|
+
* after a stall long enough to need the jitter buffer's full headroom again. `play()` gives exactly those blocks a
|
|
33
|
+
* few milliseconds' fade-in (`DISCONTINUITY_FADE_SECONDS`) through a `GainNode` to mask it; an ordinary block that
|
|
34
|
+
* picks up exactly where the last one ended gets none - fading *every* block would itself be audible, a faint
|
|
35
|
+
* tremolo at the packet rate. The decoder is created lazily and discarded on any error, and replaced by the next
|
|
36
|
+
* packet.
|
|
25
37
|
*/
|
|
26
38
|
export class AudioPlayer {
|
|
27
39
|
/** `destination` is the node whose stream the peer's `MediaStream` carries. */
|
|
@@ -87,9 +99,24 @@ export class AudioPlayer {
|
|
|
87
99
|
buffer.copyToChannel(pcm, 0);
|
|
88
100
|
const source = this.ctx.createBufferSource();
|
|
89
101
|
source.buffer = buffer;
|
|
90
|
-
source.connect(this.destination);
|
|
91
|
-
source.onended = () => source.disconnect();
|
|
92
102
|
const start = Math.max(now + JITTER_BUFFER_SECONDS, this.nextTime);
|
|
103
|
+
if (start > this.nextTime) {
|
|
104
|
+
// A gap, not a continuation - the decoder's state no longer matches what it's about to decode (see
|
|
105
|
+
// this class's doc comment), so fade this block in rather than let it start abruptly.
|
|
106
|
+
const gain = this.ctx.createGain();
|
|
107
|
+
gain.gain.setValueAtTime(0, start);
|
|
108
|
+
gain.gain.linearRampToValueAtTime(1, start + DISCONTINUITY_FADE_SECONDS);
|
|
109
|
+
source.connect(gain);
|
|
110
|
+
gain.connect(this.destination);
|
|
111
|
+
source.onended = () => {
|
|
112
|
+
source.disconnect();
|
|
113
|
+
gain.disconnect();
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
source.connect(this.destination);
|
|
118
|
+
source.onended = () => source.disconnect();
|
|
119
|
+
}
|
|
93
120
|
source.start(start);
|
|
94
121
|
this.nextTime = start + buffer.duration;
|
|
95
122
|
}
|
|
@@ -69,6 +69,11 @@ export interface ScriptProcessorLike extends AudioNodeLike {
|
|
|
69
69
|
export interface GainLike extends AudioNodeLike {
|
|
70
70
|
gain: {
|
|
71
71
|
value: number;
|
|
72
|
+
/** `AudioParam.setValueAtTime()` - used to schedule a fade (`AudioPlayer`'s discontinuity fade-in), not
|
|
73
|
+
* needed for `AudioSender`'s own static, unscheduled mute (a plain `.value = 0` assignment). */
|
|
74
|
+
setValueAtTime(value: number, startTime: number): unknown;
|
|
75
|
+
/** `AudioParam.linearRampToValueAtTime()`. */
|
|
76
|
+
linearRampToValueAtTime(value: number, endTime: number): unknown;
|
|
72
77
|
};
|
|
73
78
|
}
|
|
74
79
|
export interface AudioBufferLike {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rapidmx/meet-plugin",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.3",
|
|
4
4
|
"description": "RapidMX plugin that provides WebRTC video conferencing, joinable from a calendar invite or a shared link.",
|
|
5
5
|
"repository": "https://github.com/rapidmx/meet-plugin.git",
|
|
6
6
|
"author": "RapidREST <rapidrests@gmail.com>",
|