@rapidmx/meet-plugin 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/apps/meet/_CallControls.tsx +119 -2
  2. package/apps/meet/_CallView.tsx +59 -122
  3. package/apps/meet/_DiagnosticsPanel.tsx +65 -7
  4. package/apps/meet/_ParticipantTile.tsx +82 -9
  5. package/apps/meet/_ParticipantsDrawer.tsx +10 -7
  6. package/apps/meet/_icons.tsx +10 -0
  7. package/apps/shared/relay/AudioPlayer.ts +29 -0
  8. package/apps/shared/relay/AudioSender.ts +23 -0
  9. package/apps/shared/relay/RelayClient.ts +52 -1
  10. package/apps/shared/relay/RelayReceiver.ts +43 -2
  11. package/apps/shared/relay/RelaySender.ts +23 -1
  12. package/apps/shared/relay/RelayTransport.ts +19 -0
  13. package/apps/shared/relay/SilenceMeter.ts +59 -0
  14. package/apps/shared/webrtc/MeshConnectionManager.ts +25 -3
  15. package/apps/shared/webrtc/types.ts +71 -2
  16. package/dist/apps/meet/_CallControls.d.ts +24 -2
  17. package/dist/apps/meet/_CallControls.js +34 -3
  18. package/dist/apps/meet/_CallView.d.ts +28 -15
  19. package/dist/apps/meet/_CallView.js +50 -64
  20. package/dist/apps/meet/_DiagnosticsPanel.d.ts +5 -3
  21. package/dist/apps/meet/_DiagnosticsPanel.js +17 -1
  22. package/dist/apps/meet/_ParticipantTile.d.ts +9 -2
  23. package/dist/apps/meet/_ParticipantTile.js +36 -5
  24. package/dist/apps/meet/_ParticipantsDrawer.js +3 -3
  25. package/dist/apps/meet/_icons.d.ts +4 -0
  26. package/dist/apps/meet/_icons.js +4 -0
  27. package/dist/apps/shared/relay/AudioPlayer.d.ts +9 -0
  28. package/dist/apps/shared/relay/AudioPlayer.js +24 -4
  29. package/dist/apps/shared/relay/AudioSender.d.ts +8 -0
  30. package/dist/apps/shared/relay/AudioSender.js +20 -4
  31. package/dist/apps/shared/relay/RelayClient.d.ts +16 -0
  32. package/dist/apps/shared/relay/RelayClient.js +44 -1
  33. package/dist/apps/shared/relay/RelayReceiver.d.ts +5 -0
  34. package/dist/apps/shared/relay/RelayReceiver.js +38 -6
  35. package/dist/apps/shared/relay/RelaySender.d.ts +7 -0
  36. package/dist/apps/shared/relay/RelaySender.js +20 -5
  37. package/dist/apps/shared/relay/RelayTransport.d.ts +4 -0
  38. package/dist/apps/shared/relay/RelayTransport.js +14 -0
  39. package/dist/apps/shared/relay/SilenceMeter.d.ts +25 -0
  40. package/dist/apps/shared/relay/SilenceMeter.js +55 -0
  41. package/dist/apps/shared/webrtc/MeshConnectionManager.js +20 -1
  42. package/dist/apps/shared/webrtc/types.d.ts +66 -2
  43. package/dist/lib/util/RelayHub.js +15 -2
  44. package/dist/lib/util/RelayHub.js.map +1 -1
  45. package/dist/types/util/RelayHub.d.ts +5 -2
  46. package/package.json +1 -1
@@ -14,10 +14,13 @@
14
14
  * natural - this is a local display transform only, never applied to the video actually sent to anyone, so
15
15
  * everyone else always sees the participant the right way round. `mirrored={false}` (`_CallView.tsx`, when a custom
16
16
  * background image is on) turns that off: the picture the participant chose is a fixed reference, not a live
17
- * reflection, and mirroring it would show it backwards to no one but themselves. */
18
- import React, { useEffect, useRef } from "react";
17
+ * reflection, and mirroring it would show it backwards to no one but themselves.
18
+ *
19
+ * `onHideSelf` adds a small "…" menu to the tile's corner, local-tile-only - see `_CallView.tsx`'s doc comment on
20
+ * hiding the local tile from grid view. */
21
+ import React, { useEffect, useRef, useState } from "react";
19
22
  import type { MediaTransport } from "../shared/webrtc/types.js";
20
- import { MicOffIcon } from "./_icons.js";
23
+ import { MicOffIcon, OverflowIcon } from "./_icons.js";
21
24
 
22
25
  /** What a tile says about how its participant's media is arriving, for the paths that are not the ordinary direct one
23
26
  * (which needs no comment). Text rather than only an icon, so the reason a picture is degraded is never a guess.
@@ -74,6 +77,10 @@ export interface ParticipantTileProps {
74
77
  /** Whether the local tile is shown mirrored, like a real mirror - ignored for a remote tile, which is never
75
78
  * mirrored. Default `true`; see this module's doc comment. */
76
79
  mirrored?: boolean;
80
+ /** Offers a small "…" menu in the tile's top-right corner with one item, "Hide self" - see `_CallView.tsx`'s
81
+ * doc comment on hiding the local tile. Omitted everywhere that shouldn't offer it: every remote tile, and the
82
+ * local tile while alone in the call, where hiding it would just blank the only thing on screen. */
83
+ onHideSelf?: () => void;
77
84
  onClick?: () => void;
78
85
  className?: string;
79
86
  }
@@ -94,12 +101,15 @@ export default function ParticipantTile({
94
101
  isFocused,
95
102
  contain,
96
103
  mirrored = true,
104
+ onHideSelf,
97
105
  onClick,
98
106
  className,
99
107
  }: ParticipantTileProps) {
100
108
  const badge = transport ? TRANSPORT_BADGES[transport] : undefined;
101
109
  const videoRef = useRef<HTMLVideoElement>(null);
102
110
  const showVideo = !!stream && !cameraOff;
111
+ const [hideMenuOpen, setHideMenuOpen] = useState(false);
112
+ const hideMenuRef = useRef<HTMLDivElement>(null);
103
113
 
104
114
  // The `<video>` element only exists while `showVideo`, so its stream is bound whenever it (re)appears too.
105
115
  useEffect(() => {
@@ -108,6 +118,30 @@ export default function ParticipantTile({
108
118
  }
109
119
  }, [stream, showVideo]);
110
120
 
121
+ // The tile's own "…" menu closes on Escape or a press anywhere outside it, same shape as every other menu in
122
+ // this app.
123
+ useEffect(() => {
124
+ if (!hideMenuOpen) {
125
+ return;
126
+ }
127
+ const onPointerDown = (event: PointerEvent) => {
128
+ if (!hideMenuRef.current?.contains(event.target as Node)) {
129
+ setHideMenuOpen(false);
130
+ }
131
+ };
132
+ const onKeyDown = (event: KeyboardEvent) => {
133
+ if (event.key === "Escape") {
134
+ setHideMenuOpen(false);
135
+ }
136
+ };
137
+ document.addEventListener("pointerdown", onPointerDown);
138
+ document.addEventListener("keydown", onKeyDown);
139
+ return () => {
140
+ document.removeEventListener("pointerdown", onPointerDown);
141
+ document.removeEventListener("keydown", onKeyDown);
142
+ };
143
+ }, [hideMenuOpen]);
144
+
111
145
  return (
112
146
  <div
113
147
  className={`relative overflow-hidden rounded-xl bg-[#3c4043] flex items-center justify-center min-h-[90px] ${
@@ -142,12 +176,51 @@ export default function ParticipantTile({
142
176
  ✋
143
177
  </span>
144
178
  )}
145
- {micMuted && (
146
- <span className="absolute top-2 right-2 flex items-center justify-center w-7 h-7 rounded-full bg-black/60 text-white" role="img" aria-label="Muted">
147
- <span className="w-4 h-4 [&>svg]:w-4 [&>svg]:h-4">
148
- <MicOffIcon />
149
- </span>
150
- </span>
179
+ {(micMuted || onHideSelf) && (
180
+ <div className="absolute top-2 right-2 flex items-center gap-1">
181
+ {micMuted && (
182
+ <span className="flex items-center justify-center w-7 h-7 rounded-full bg-black/60 text-white" role="img" aria-label="Muted">
183
+ <span className="w-4 h-4 [&>svg]:w-4 [&>svg]:h-4">
184
+ <MicOffIcon />
185
+ </span>
186
+ </span>
187
+ )}
188
+ {onHideSelf && (
189
+ <div className="relative" ref={hideMenuRef}>
190
+ <button
191
+ type="button"
192
+ className="flex items-center justify-center w-7 h-7 rounded-full bg-black/60 hover:bg-black/80 text-white focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80"
193
+ aria-label="Local video options"
194
+ aria-haspopup="menu"
195
+ aria-expanded={hideMenuOpen}
196
+ onClick={() => setHideMenuOpen((prev) => !prev)}
197
+ >
198
+ <span className="w-4 h-4 [&>svg]:w-4 [&>svg]:h-4">
199
+ <OverflowIcon />
200
+ </span>
201
+ </button>
202
+ {hideMenuOpen && (
203
+ <div
204
+ role="menu"
205
+ aria-label="Local video options"
206
+ className="absolute top-full right-0 mt-1 w-32 rounded-xl bg-[#2b2d30] text-white shadow-xl py-1 text-sm"
207
+ >
208
+ <button
209
+ type="button"
210
+ role="menuitem"
211
+ className="w-full px-3 py-1.5 text-left hover:bg-white/10 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80"
212
+ onClick={() => {
213
+ setHideMenuOpen(false);
214
+ onHideSelf();
215
+ }}
216
+ >
217
+ Hide self
218
+ </button>
219
+ </div>
220
+ )}
221
+ </div>
222
+ )}
223
+ </div>
151
224
  )}
152
225
  {badge && (
153
226
  <span
@@ -35,7 +35,7 @@ import React, { useEffect } from "react";
35
35
  import type { MeshParticipant } from "../shared/webrtc/types.js";
36
36
  import type { WaitingParticipant } from "./_meetApi.js";
37
37
  import { TRANSPORT_BADGES } from "./_ParticipantTile.js";
38
- import { MicOffIcon } from "./_icons.js";
38
+ import { BatonIcon, MicOffIcon, RemoveIcon } from "./_icons.js";
39
39
 
40
40
  export interface ParticipantsDrawerProps {
41
41
  selfName: string;
@@ -184,35 +184,38 @@ function Row({
184
184
  {canGiveStick && !hasStick && (
185
185
  <button
186
186
  type="button"
187
- className="px-2 py-0.5 rounded text-xs whitespace-nowrap bg-white/10 hover:bg-white/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80"
187
+ className="w-7 h-7 flex items-center justify-center rounded-full bg-white/10 hover:bg-white/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80 [&>svg]:w-4 [&>svg]:h-4"
188
188
  aria-label={`Give the talking stick to ${name}`}
189
+ title={`Give the talking stick to ${name}`}
189
190
  onClick={onGiveStick}
190
191
  >
191
- Give stick
192
+ <BatonIcon />
192
193
  </button>
193
194
  )}
194
195
  {canModerate && !micMuted && (
195
196
  <button
196
197
  type="button"
197
- className="px-2 py-0.5 rounded text-xs whitespace-nowrap bg-white/10 hover:bg-white/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80"
198
+ className="w-7 h-7 flex items-center justify-center rounded-full bg-white/10 hover:bg-white/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80 [&>svg]:w-4 [&>svg]:h-4"
198
199
  aria-label={`Mute ${name}`}
200
+ title={`Mute ${name}`}
199
201
  onClick={onMute}
200
202
  >
201
- Mute
203
+ <MicOffIcon />
202
204
  </button>
203
205
  )}
204
206
  {canModerate && (
205
207
  <button
206
208
  type="button"
207
- className="px-2 py-0.5 rounded text-xs whitespace-nowrap bg-[#601410] text-[#f9dedc] hover:bg-[#7a1b16] focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80"
209
+ className="w-7 h-7 flex items-center justify-center rounded-full bg-[#601410] text-[#f9dedc] hover:bg-[#7a1b16] focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80 [&>svg]:w-4 [&>svg]:h-4"
208
210
  aria-label={`Remove ${name}`}
211
+ title={`Remove ${name}`}
209
212
  onClick={() => {
210
213
  if (window.confirm(`Remove ${name} from the call?`)) {
211
214
  onKick?.();
212
215
  }
213
216
  }}
214
217
  >
215
- Remove
218
+ <RemoveIcon />
216
219
  </button>
217
220
  )}
218
221
  </li>
@@ -78,3 +78,13 @@ export const BatonIcon = () => (
78
78
  );
79
79
 
80
80
  export const OverflowIcon = () => <Icon path="M6 10c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2zm12 0c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2zm-6 0c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2z" />;
81
+
82
+ /** A plain "X" - removing a participant from the call (the participants drawer's "Remove" row button). Drawn with
83
+ * strokes rather than a filled Material path, same "simple primitives, not a recalled path" reasoning as
84
+ * `BatonIcon`. */
85
+ export const RemoveIcon = () => (
86
+ <svg viewBox="0 0 24 24" className="w-6 h-6" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" aria-hidden="true" focusable="false">
87
+ <line x1="6" y1="6" x2="18" y2="18" />
88
+ <line x1="18" y1="6" x2="6" y2="18" />
89
+ </svg>
90
+ );
@@ -2,9 +2,14 @@
2
2
  // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
3
  // SPDX-License-Identifier: MPL-2.0
4
4
  ///////////////////////////////////////////////////////////////////////////////
5
+ import type { RelayReceiveDiagnostics } from "../webrtc/types.js";
5
6
  import { TimestampUnwrapper, type Frame } from "./frames.js";
6
7
  import type { AudioContextLike, AudioDataLike, DecoderLike, RelayEnv } from "./relayEnv.js";
7
8
  import { safeClose } from "./senderCommon.js";
9
+ import { SilenceMeter } from "./SilenceMeter.js";
10
+
11
+ /** The playback half of `RelayReceiveDiagnostics.audio`. */
12
+ export type AudioPlaybackStats = Pick<RelayReceiveDiagnostics["audio"], "playedMs" | "silentMs" | "gaps" | "gapMs" | "droppedLate" | "bufferedMs">;
8
13
 
9
14
  export const PLAYBACK_SAMPLE_RATE = 48_000;
10
15
  /** How far ahead of the clock the first packet after a start or an underrun is scheduled. It is the jitter buffer:
@@ -42,6 +47,10 @@ export class AudioPlayer {
42
47
  private decoder: DecoderLike | undefined;
43
48
  private nextTime = 0;
44
49
  private readonly timestamps = new TimestampUnwrapper();
50
+ private readonly played = new SilenceMeter();
51
+ private gaps = 0;
52
+ private gapSeconds = 0;
53
+ private droppedLate = 0;
45
54
 
46
55
  /** `destination` is the node whose stream the peer's `MediaStream` carries. */
47
56
  constructor(
@@ -50,10 +59,23 @@ export class AudioPlayer {
50
59
  private readonly destination: unknown,
51
60
  ) {}
52
61
 
62
+ /** What playback has done so far - see `RelayReceiveDiagnostics`. */
63
+ stats(): AudioPlaybackStats {
64
+ return {
65
+ playedMs: this.played.totalMs,
66
+ silentMs: this.played.silentMs,
67
+ gaps: this.gaps,
68
+ gapMs: this.gapSeconds * 1000,
69
+ droppedLate: this.droppedLate,
70
+ bufferedMs: Math.max(0, this.nextTime - this.ctx.currentTime) * 1000,
71
+ };
72
+ }
73
+
53
74
  /** Feeds one reassembled audio frame (one Opus packet). */
54
75
  push(frame: Frame): void {
55
76
  const decoder = this.decoder ?? this.createDecoder();
56
77
  if (!decoder || decoder.decodeQueueSize > AUDIO_MAX_DECODE_QUEUE) {
78
+ this.droppedLate += 1;
57
79
  return;
58
80
  }
59
81
  try {
@@ -101,6 +123,7 @@ export class AudioPlayer {
101
123
  try {
102
124
  const now = this.ctx.currentTime;
103
125
  if (this.nextTime - now > MAX_AHEAD_SECONDS) {
126
+ this.droppedLate += 1;
104
127
  return;
105
128
  }
106
129
  const buffer = this.ctx.createBuffer(1, data.numberOfFrames, data.sampleRate);
@@ -127,6 +150,12 @@ export class AudioPlayer {
127
150
  source.onended = () => source.disconnect();
128
151
  }
129
152
  source.start(start);
153
+ this.played.feed(pcm, data.sampleRate);
154
+ if (start > this.nextTime && this.nextTime > 0) {
155
+ // Playback ran dry: everything between the end of the last block and this one is silence.
156
+ this.gaps += 1;
157
+ this.gapSeconds += start - this.nextTime;
158
+ }
130
159
  this.nextTime = start + buffer.duration;
131
160
  } catch {
132
161
  // A block the browser will not take (an odd sample rate, a closed context) is dropped like a lost packet.
@@ -2,6 +2,7 @@
2
2
  // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
3
  // SPDX-License-Identifier: MPL-2.0
4
4
  ///////////////////////////////////////////////////////////////////////////////
5
+ import type { RelaySendDiagnostics } from "../webrtc/types.js";
5
6
  import { resumeAudioContext } from "./audioResume.js";
6
7
  import { KIND_AUDIO } from "./frames.js";
7
8
  import type {
@@ -14,6 +15,7 @@ import type {
14
15
  ScriptProcessorLike,
15
16
  } from "./relayEnv.js";
16
17
  import { safeClose, type SenderDeps } from "./senderCommon.js";
18
+ import { SilenceMeter } from "./SilenceMeter.js";
17
19
 
18
20
  export const AUDIO_SAMPLE_RATE = 48_000;
19
21
  /** Frames per `ScriptProcessorNode` callback: about 43 ms, a compromise between latency and how often the main
@@ -53,8 +55,24 @@ export class AudioSender {
53
55
  private baseMs = 0;
54
56
  private samples = 0;
55
57
 
58
+ /** Diagnostics, cumulative across capture sessions: what the capture path delivered (and how much of it was
59
+ * silent), the wall-clock time it was attached for, and what the encoder had to skip. */
60
+ private readonly captured = new SilenceMeter();
61
+ private wallMs = 0;
62
+ private encoderSkippedSamples = 0;
63
+
56
64
  constructor(private readonly deps: SenderDeps) {}
57
65
 
66
+ /** The capture half of `RelaySendDiagnostics`. */
67
+ captureStats(): Pick<RelaySendDiagnostics, "audioCapturedMs" | "audioCaptureWallMs" | "audioSilentMs" | "audioEncoderSkippedMs"> {
68
+ return {
69
+ audioCapturedMs: this.captured.totalMs,
70
+ audioCaptureWallMs: this.wallMs + (this.attachedTrack ? this.deps.env.now() - this.baseMs : 0),
71
+ audioSilentMs: this.captured.silentMs,
72
+ audioEncoderSkippedMs: (this.encoderSkippedSamples * 1000) / AUDIO_SAMPLE_RATE,
73
+ };
74
+ }
75
+
58
76
  setActive(active: boolean): void {
59
77
  this.active = active;
60
78
  this.sync();
@@ -105,6 +123,9 @@ export class AudioSender {
105
123
  }
106
124
 
107
125
  private detach(): void {
126
+ if (this.attachedTrack) {
127
+ this.wallMs += this.deps.env.now() - this.baseMs;
128
+ }
108
129
  this.attachedTrack = null;
109
130
  this.stopResume?.();
110
131
  this.stopResume = undefined;
@@ -132,11 +153,13 @@ export class AudioSender {
132
153
  const samples = input.length;
133
154
  const timestampUs = Math.round((this.samples * 1_000_000) / AUDIO_SAMPLE_RATE);
134
155
  this.samples += samples;
156
+ this.captured.feed(input, AUDIO_SAMPLE_RATE);
135
157
  if (!this.deps.canSend()) {
136
158
  return;
137
159
  }
138
160
  const encoder = this.encoder ?? this.createEncoder();
139
161
  if (!encoder || encoder.encodeQueueSize > AUDIO_MAX_ENCODE_QUEUE) {
162
+ this.encoderSkippedSamples += samples;
140
163
  return;
141
164
  }
142
165
  let audio: AudioDataLike | undefined;
@@ -13,6 +13,10 @@
13
13
  * older one, which accepts `MAX_MESSAGE_BYTES`). Nothing else may be sent before `ready`.
14
14
  * - `{"op":"want","peers":[...]}` REPLACES the set of senders this socket receives from (the server accepts at most
15
15
  * `MAX_WANTED_PEERS`). It has to be re-sent after every reconnect, because the server's set dies with the socket.
16
+ * - `{"op":"ping","t":<this client's clock>}`, every `RELAY_PING_INTERVAL_MS` once ready, is answered with
17
+ * `{"op":"pong","t":<the same>}`, which times the round trip to the server (`roundTripMs`) - over the same socket as
18
+ * the media, so it includes whatever the media has queued ahead of it. An older server ignores the ping and the
19
+ * round trip stays unknown.
16
20
  * - A binary message from the client is an opaque payload of at most `maxMessageBytes`. The server forwards it to
17
21
  * every socket that wants this sender as `[1 byte N][N bytes UTF-8 sender peer id][payload]`, stamping the sender
18
22
  * itself so a peer cannot speak as another.
@@ -48,6 +52,9 @@ export const RELAY_BACKOFF_MAX_MS = 15_000;
48
52
  /** How long a socket may take to become `ready` before it is abandoned and retried (a server that accepts the
49
53
  * upgrade but never answers the hello would otherwise hang the tier forever). */
50
54
  export const RELAY_HANDSHAKE_TIMEOUT_MS = 10_000;
55
+ /** How often a ready socket pings the server to time the round trip - often enough that the diagnostics panel (which
56
+ * polls every few seconds) always has a recent sample, rarely enough to cost nothing next to the media. */
57
+ export const RELAY_PING_INTERVAL_MS = 2_000;
51
58
 
52
59
  const SOCKET_OPEN = 1;
53
60
 
@@ -56,6 +63,8 @@ export interface RelayClientOptions {
56
63
  peerId: string;
57
64
  createSocket: (url: string) => RelaySocket;
58
65
  random: () => number;
66
+ /** A monotonic millisecond clock for timing pings. Defaults to `Date.now`. */
67
+ now?: () => number;
59
68
  /** Called with the sender's peer id (as stamped by the server) and the original payload. */
60
69
  onMedia: (peerId: string, payload: Uint8Array) => void;
61
70
  }
@@ -82,6 +91,8 @@ export class RelayClient {
82
91
  private wanted: string[] = [];
83
92
  private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
84
93
  private handshakeTimer: ReturnType<typeof setTimeout> | undefined;
94
+ private pingTimer: ReturnType<typeof setInterval> | undefined;
95
+ private lastRoundTripMs: number | undefined;
85
96
  private readonly decoder = new TextDecoder();
86
97
 
87
98
  constructor(private readonly options: RelayClientOptions) {}
@@ -91,6 +102,12 @@ export class RelayClient {
91
102
  return this.isReady;
92
103
  }
93
104
 
105
+ /** The most recent ping's round trip to the server, in milliseconds - `undefined` until the first pong on the
106
+ * current socket (and for good on a server too old to answer pings). */
107
+ get roundTripMs(): number | undefined {
108
+ return this.lastRoundTripMs;
109
+ }
110
+
94
111
  /** The largest message `sendMedia()` will send: what the server announced in `ready` (validated and clamped), or
95
112
  * `MAX_MESSAGE_BYTES` before that, without one, and again after every disconnect until the next `ready`. The
96
113
  * frame fragmenter sizes its fragments from this. */
@@ -148,6 +165,7 @@ export class RelayClient {
148
165
  clearTimeout(this.reconnectTimer);
149
166
  clearTimeout(this.handshakeTimer);
150
167
  this.reconnectTimer = this.handshakeTimer = undefined;
168
+ this.stopPinging();
151
169
  this.detach();
152
170
  }
153
171
 
@@ -200,7 +218,15 @@ export class RelayClient {
200
218
  } catch {
201
219
  return;
202
220
  }
203
- if (!message || typeof message !== "object" || (message as { op?: unknown }).op !== "ready" || this.isReady) {
221
+ if (!message || typeof message !== "object") {
222
+ return;
223
+ }
224
+ const op = (message as { op?: unknown }).op;
225
+ if (op === "pong") {
226
+ this.handlePong((message as { t?: unknown }).t);
227
+ return;
228
+ }
229
+ if (op !== "ready" || this.isReady) {
204
230
  return;
205
231
  }
206
232
  this.isReady = true;
@@ -213,6 +239,30 @@ export class RelayClient {
213
239
  this.handshakeTimer = undefined;
214
240
  // The server's wanted set is per socket, so it starts empty on every (re)connect.
215
241
  this.sendText({ op: "want", peers: this.wanted });
242
+ this.pingTimer = setInterval(() => this.sendText({ op: "ping", t: this.now() }), RELAY_PING_INTERVAL_MS);
243
+ }
244
+
245
+ /** Times the round trip of the ping that `t` came back from. Ignored before ready, for a `t` that isn't a number,
246
+ * or one from the future (nothing this client sent). */
247
+ private handlePong(t: unknown): void {
248
+ if (!this.isReady || typeof t !== "number") {
249
+ return;
250
+ }
251
+ const roundTrip = this.now() - t;
252
+ if (roundTrip >= 0) {
253
+ this.lastRoundTripMs = roundTrip;
254
+ }
255
+ }
256
+
257
+ private now(): number {
258
+ return (this.options.now ?? Date.now)();
259
+ }
260
+
261
+ /** Stops pinging and forgets the last round trip - it described a socket that is gone. */
262
+ private stopPinging(): void {
263
+ clearInterval(this.pingTimer);
264
+ this.pingTimer = undefined;
265
+ this.lastRoundTripMs = undefined;
216
266
  }
217
267
 
218
268
  private handleClose(socket: RelaySocket): void {
@@ -220,6 +270,7 @@ export class RelayClient {
220
270
  return;
221
271
  }
222
272
  this.isReady = false;
273
+ this.stopPinging();
223
274
  // The next server might be a different one (a rolling upgrade), so the limit is learned afresh.
224
275
  this.messageLimit = MAX_MESSAGE_BYTES;
225
276
  clearTimeout(this.handshakeTimer);
@@ -2,11 +2,38 @@
2
2
  // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
3
  // SPDX-License-Identifier: MPL-2.0
4
4
  ///////////////////////////////////////////////////////////////////////////////
5
+ import type { RelayReceiveDiagnostics, RelayReceiveStats } from "../webrtc/types.js";
5
6
  import { AudioPlayer } from "./AudioPlayer.js";
6
- import { decodeFragment, FrameReassembler, KIND_AUDIO } from "./frames.js";
7
+ import { decodeFragment, type Frame, FrameReassembler, KIND_AUDIO, seqIsNewer } from "./frames.js";
7
8
  import type { AudioContextLike, RelayEnv } from "./relayEnv.js";
8
9
  import { VideoPlayer } from "./VideoPlayer.js";
9
10
 
11
+ const SEQ_MODULUS = 0x10000;
12
+ /** A jump in sequence numbers bigger than this is taken for the sender restarting its counter, not for that many
13
+ * lost frames. Ten seconds of 20 ms audio packets - far more than any real outage the stream would survive. */
14
+ const MAX_COUNTED_SEQ_GAP = 500;
15
+
16
+ /** Counts one kind's frames as they complete, and the ones the sequence numbers say never did. */
17
+ class ReceiveCounter {
18
+ readonly stats: RelayReceiveStats = { framesReceived: 0, framesLost: 0, bytesReceived: 0 };
19
+ private lastSeq: number | undefined;
20
+
21
+ fragment(bytes: number): void {
22
+ this.stats.bytesReceived += bytes;
23
+ }
24
+
25
+ frame(frame: Frame): void {
26
+ this.stats.framesReceived += 1;
27
+ if (this.lastSeq !== undefined && seqIsNewer(frame.seq, this.lastSeq)) {
28
+ const missing = ((frame.seq - this.lastSeq + SEQ_MODULUS) % SEQ_MODULUS) - 1;
29
+ if (missing <= MAX_COUNTED_SEQ_GAP) {
30
+ this.stats.framesLost += missing;
31
+ }
32
+ }
33
+ this.lastSeq = frame.seq;
34
+ }
35
+ }
36
+
10
37
  /** The canvas is sized to the first decoded picture; until then it is this size. */
11
38
  const INITIAL_CANVAS_WIDTH = 320;
12
39
  const INITIAL_CANVAS_HEIGHT = 240;
@@ -25,6 +52,8 @@ export class RelayReceiver {
25
52
  readonly stream: MediaStream;
26
53
  private readonly audioFrames = new FrameReassembler();
27
54
  private readonly videoFrames = new FrameReassembler();
55
+ private readonly audioCounter = new ReceiveCounter();
56
+ private readonly videoCounter = new ReceiveCounter();
28
57
  private readonly audio: AudioPlayer;
29
58
  private readonly video: VideoPlayer | undefined;
30
59
 
@@ -50,10 +79,14 @@ export class RelayReceiver {
50
79
  if (!fragment) {
51
80
  return;
52
81
  }
53
- const frame = (fragment.kind === KIND_AUDIO ? this.audioFrames : this.videoFrames).push(fragment);
82
+ const isAudio = fragment.kind === KIND_AUDIO;
83
+ const counter = isAudio ? this.audioCounter : this.videoCounter;
84
+ counter.fragment(payload.length);
85
+ const frame = (isAudio ? this.audioFrames : this.videoFrames).push(fragment);
54
86
  if (!frame) {
55
87
  return;
56
88
  }
89
+ counter.frame(frame);
57
90
  if (frame.kind === KIND_AUDIO) {
58
91
  this.audio.push(frame);
59
92
  } else {
@@ -61,6 +94,14 @@ export class RelayReceiver {
61
94
  }
62
95
  }
63
96
 
97
+ /** A snapshot of what has been received from this peer - see `RelayReceiveDiagnostics`. */
98
+ diagnostics(): RelayReceiveDiagnostics {
99
+ return {
100
+ audio: { ...this.audioCounter.stats, ...this.audio.stats() },
101
+ video: { ...this.videoCounter.stats },
102
+ };
103
+ }
104
+
64
105
  /** Releases the decoders and ends the stream's tracks. */
65
106
  close(): void {
66
107
  this.audio.close();
@@ -2,6 +2,7 @@
2
2
  // Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
3
3
  // SPDX-License-Identifier: MPL-2.0
4
4
  ///////////////////////////////////////////////////////////////////////////////
5
+ import type { RelaySendDiagnostics, RelaySendStats } from "../webrtc/types.js";
5
6
  import { AudioSender } from "./AudioSender.js";
6
7
  import { DEFAULT_MESSAGE_BYTES, fragmentFrame, type FrameKind, KIND_AUDIO, KIND_VIDEO } from "./frames.js";
7
8
  import type { RelayEnv } from "./relayEnv.js";
@@ -15,6 +16,11 @@ const SEQ_MODULUS = 0x10000;
15
16
  * otherwise treat the restarted stream as older than what it has already seen. */
16
17
  export class FramePublisher {
17
18
  private readonly seqs: Record<FrameKind, number> = { [KIND_AUDIO]: 0, [KIND_VIDEO]: 0 };
19
+ /** What happened to each kind's frames - for the diagnostics panel. */
20
+ readonly stats: Record<FrameKind, RelaySendStats> = {
21
+ [KIND_AUDIO]: { framesSent: 0, framesDropped: 0, bytesSent: 0 },
22
+ [KIND_VIDEO]: { framesSent: 0, framesDropped: 0, bytesSent: 0 },
23
+ };
18
24
 
19
25
  /** `send` returns whether one message was accepted by the socket. */
20
26
  /** `maxMessageBytes` is the largest message the socket may send right now (it changes when the server's `ready`
@@ -29,15 +35,20 @@ export class FramePublisher {
29
35
  publish: PublishFrame = (kind, keyFrame, data, timestampMs) => {
30
36
  const seq = this.seqs[kind];
31
37
  this.seqs[kind] = (seq + 1) % SEQ_MODULUS;
38
+ const stats = this.stats[kind];
32
39
  const fragments = fragmentFrame(kind, keyFrame, seq, timestampMs, data, this.maxMessageBytes());
33
40
  if (fragments.length === 0) {
41
+ stats.framesDropped += 1;
34
42
  return false;
35
43
  }
36
44
  for (const fragment of fragments) {
37
45
  if (!this.send(fragment)) {
46
+ stats.framesDropped += 1;
38
47
  return false;
39
48
  }
49
+ stats.bytesSent += fragment.length;
40
50
  }
51
+ stats.framesSent += 1;
41
52
  return true;
42
53
  };
43
54
  }
@@ -52,15 +63,26 @@ export interface RelaySenderOptions {
52
63
 
53
64
  /** The publishing half of the relay: the two capture paths plus the frame numbering they share. */
54
65
  export class RelaySender {
66
+ private readonly publisher: FramePublisher;
55
67
  private readonly audio: AudioSender;
56
68
  private readonly video: VideoSender;
57
69
 
58
70
  constructor(options: RelaySenderOptions) {
59
- const deps = { env: options.env, publish: new FramePublisher(options.send, options.maxMessageBytes).publish, canSend: options.canSend };
71
+ this.publisher = new FramePublisher(options.send, options.maxMessageBytes);
72
+ const deps = { env: options.env, publish: this.publisher.publish, canSend: options.canSend };
60
73
  this.audio = new AudioSender(deps);
61
74
  this.video = new VideoSender(deps);
62
75
  }
63
76
 
77
+ /** A snapshot of the sending counters - see `RelaySendDiagnostics`. */
78
+ diagnostics(): RelaySendDiagnostics {
79
+ return {
80
+ ...this.audio.captureStats(),
81
+ audio: { ...this.publisher.stats[KIND_AUDIO] },
82
+ video: { ...this.publisher.stats[KIND_VIDEO] },
83
+ };
84
+ }
85
+
64
86
  /** Starts or stops encoding whatever tracks are set. Stopping releases the encoders and audio nodes. */
65
87
  setActive(active: boolean): void {
66
88
  this.audio.setActive(active);
@@ -17,6 +17,7 @@
17
17
  * negotiation, because both ends are this same code.
18
18
  */
19
19
  import { apiOrigin } from "@rapidmx/web-client/lib/util/api.js";
20
+ import type { RelayDiagnostics } from "../webrtc/types.js";
20
21
  import { PLAYBACK_SAMPLE_RATE } from "./AudioPlayer.js";
21
22
  import { resumeAudioContext } from "./audioResume.js";
22
23
  import { RelayClient } from "./RelayClient.js";
@@ -39,6 +40,9 @@ export interface RelayTransport {
39
40
  /** Same semantics as `MeshConnectionManager.setLocalTrack`: null = send nothing for that kind. Safe to call
40
41
  * before or after `setSending`. */
41
42
  setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
43
+ /** The sending counters (shared by every peer) plus what has been received from `peerId`, for the diagnostics
44
+ * panel - cumulative, so a caller wanting a rate keeps the previous sample. */
45
+ diagnostics(peerId: string): RelayDiagnostics;
42
46
  /** Closes the socket, all coders, streams and timers. Idempotent. */
43
47
  close(): void;
44
48
  }
@@ -68,6 +72,16 @@ const UNSUPPORTED_TRANSPORT: RelayTransport = {
68
72
  stopReceivingFrom: () => undefined,
69
73
  setSending: () => undefined,
70
74
  setLocalTrack: () => undefined,
75
+ diagnostics: () => ({
76
+ send: {
77
+ audioCapturedMs: 0,
78
+ audioCaptureWallMs: 0,
79
+ audioSilentMs: 0,
80
+ audioEncoderSkippedMs: 0,
81
+ audio: { framesSent: 0, framesDropped: 0, bytesSent: 0 },
82
+ video: { framesSent: 0, framesDropped: 0, bytesSent: 0 },
83
+ },
84
+ }),
71
85
  close: () => undefined,
72
86
  };
73
87
 
@@ -90,6 +104,7 @@ class WebSocketRelayTransport implements RelayTransport {
90
104
  peerId,
91
105
  createSocket: (target) => env.createSocket(target),
92
106
  random: () => env.random(),
107
+ now: () => env.now(),
93
108
  onMedia: (sender, payload) => this.receivers.get(sender)?.handleMedia(payload),
94
109
  });
95
110
  this.sender = new RelaySender({
@@ -140,6 +155,10 @@ class WebSocketRelayTransport implements RelayTransport {
140
155
  }
141
156
  }
142
157
 
158
+ diagnostics(peerId: string): RelayDiagnostics {
159
+ return { roundTripMs: this.client.roundTripMs, send: this.sender.diagnostics(), receive: this.receivers.get(peerId)?.diagnostics() };
160
+ }
161
+
143
162
  close(): void {
144
163
  if (this.closed) {
145
164
  return;