@voltkit-rtc/client 0.7.3 → 0.9.0

package/src/effects-gl.ts

@@ -0,0 +1,384 @@
+/**
+ * The WebGL2 side of video effects (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-5): the camera frame and the
+ * person mask in, the finished picture out on one canvas. Done on the GPU as LiveKit, Stream, Twilio and TRTC do:
+ *
+ * - **Blur** at a quarter of the size (cheap at any resolution), weighted by the mask so the person's own colours never
+ *   bleed into the blur around them -- no halo (Stream's mask-weighted blur) -- in two separable Gaussian passes.
+ * - **Edges** follow the person's outline rather than the mask's coarse grid: the mask is sampled jointly with the
+ *   frame's colours (a joint bilateral filter, as Stream and Twilio refine theirs).
+ * - **Background picture** fitted like `object-fit: cover`.
+ * - **Beauty** without a face model: skin found by its colour (YCbCr), smoothed with an edge-preserving filter so eyes
+ *   and lips stay sharp, brightened in the midtones, a touch rosier, and detail off the skin sharpened.
+ *
+ * Every texture is kept in picture order (the first row is the top); only the last pass, to the canvas, flips.
+ */
+import type { BeautySettings } from './effects-core.js';
+
+const VERTEX = `#version 300 es
+in vec2 position;
+uniform float flipY;
+out vec2 uv;
+void main() {
+  uv = vec2(position.x * 0.5 + 0.5, flipY > 0.5 ? 0.5 - position.y * 0.5 : 0.5 + position.y * 0.5);
+  gl_Position = vec4(position, 0.0, 1.0);
+}`;
+
+/** The frame at a quarter size, premultiplied by how much of each pixel is background. */
+const DOWNSAMPLE = `#version 300 es
+precision mediump float;
+in vec2 uv;
+out vec4 color;
+uniform sampler2D frame;
+uniform sampler2D mask;
+uniform float maskReady;
+uniform vec2 edge;
+void main() {
+  vec3 c = texture(frame, uv).rgb;
+  float person = maskReady > 0.5 ? smoothstep(edge.x, edge.y, texture(mask, uv).r) : 0.0;
+  float background = 1.0 - person;
+  color = vec4(c * background, background);
+}`;
+
+/** One direction of a Gaussian blur, on premultiplied colour. */
+const BLUR = `#version 300 es
+precision mediump float;
+in vec2 uv;
+out vec4 color;
+uniform sampler2D source;
+uniform vec2 direction;
+uniform float sigma;
+void main() {
+  vec4 sum = texture(source, uv);
+  float total = 1.0;
+  for (int i = 1; i <= 32; i++) {
+    float x = float(i);
+    if (x > sigma * 3.0) break;
+    float w = exp(-x * x / (2.0 * sigma * sigma));
+    sum += (texture(source, uv + direction * x) + texture(source, uv - direction * x)) * w;
+    total += 2.0 * w;
+  }
+  color = sum / total;
+}`;
+
+const COMPOSITE = `#version 300 es
+precision highp float;
+in vec2 uv;
+out vec4 color;
+uniform sampler2D frame;
+uniform sampler2D mask;
+uniform sampler2D blurred;
+uniform sampler2D picture;
+uniform int mode;
+uniform float maskReady;
+uniform vec2 edge;
+uniform vec2 texel;
+uniform vec2 maskTexel;
+uniform vec4 cover;
+uniform vec4 beauty;
+uniform float beautyOn;
+uniform float beautyRadius;
+
+const vec2 RING[8] = vec2[8](vec2(1.0, 0.0), vec2(0.7071, 0.7071), vec2(0.0, 1.0), vec2(-0.7071, 0.7071),
+                             vec2(-1.0, 0.0), vec2(-0.7071, -0.7071), vec2(0.0, -1.0), vec2(0.7071, -0.7071));
+
+float skinOf(vec3 c) {
+  float y = dot(c, vec3(0.299, 0.587, 0.114));
+  float cb = 0.5 + (c.b - y) * 0.564;
+  float cr = 0.5 + (c.r - y) * 0.713;
+  float s = smoothstep(0.27, 0.31, cb) * (1.0 - smoothstep(0.49, 0.53, cb))
+          * smoothstep(0.51, 0.55, cr) * (1.0 - smoothstep(0.67, 0.71, cr));
+  return s * smoothstep(0.06, 0.15, y);
+}
+
+vec3 beautify(vec3 c0) {
+  vec3 sum = c0;
+  float total = 1.0;
+  for (int r = 1; r <= 2; r++) {
+    float radius = beautyRadius * float(r);
+    float near = r == 1 ? 0.8 : 0.5;
+    for (int i = 0; i < 8; i++) {
+      vec3 c = texture(frame, uv + RING[i] * radius * texel).rgb;
+      vec3 d = c - c0;
+      float w = near * exp(-dot(d, d) * 60.0);
+      sum += c * w;
+      total += w;
+    }
+  }
+  vec3 smoothed = sum / total;
+  float skin = skinOf(c0);
+  vec3 c = mix(c0, smoothed, beauty.x * skin);
+  c += (c0 - smoothed) * beauty.w * (1.0 - skin) * 1.5;
+  float lift = beauty.y * (0.35 + 0.65 * skin) * 0.5;
+  c += lift * c * (1.0 - c);
+  c = mix(c, c * vec3(1.06, 0.97, 0.98), beauty.z * skin);
+  return clamp(c, 0.0, 1.0);
+}
+
+float personAt(vec3 center) {
+  float sum = 0.0;
+  float total = 0.0;
+  for (int y = -1; y <= 1; y++) {
+    for (int x = -1; x <= 1; x++) {
+      vec2 o = vec2(float(x), float(y)) * maskTexel * 1.5;
+      vec3 d = texture(frame, uv + o).rgb - center;
+      float w = exp(-dot(d, d) * 40.0);
+      sum += texture(mask, uv + o).r * w;
+      total += w;
+    }
+  }
+  return smoothstep(edge.x, edge.y, sum / total);
+}
+
+void main() {
+  vec3 original = texture(frame, uv).rgb;
+  vec3 fg = beautyOn > 0.5 ? beautify(original) : original;
+  if (mode == 0) {
+    color = vec4(fg, 1.0);
+    return;
+  }
+  float person = maskReady > 0.5 ? personAt(original) : 0.0;
+  vec3 bg;
+  if (mode == 1) {
+    vec4 b = texture(blurred, uv);
+    bg = b.a > 0.02 ? b.rgb / b.a : fg;
+  } else {
+    bg = texture(picture, uv * cover.xy + cover.zw).rgb;
+  }
+  color = vec4(mix(bg, fg, person), 1.0);
+}`;
+
+export interface RenderSettings {
+  mode: 'none' | 'blur' | 'image';
+  /** Gaussian sigma in quarter-size texels. */
+  blurSigma: number;
+  beauty: BeautySettings | null;
+  /** False until the first mask: everything is treated as background, so nothing behind the person shows meanwhile. */
+  maskReady: boolean;
+  /** Where the image texture maps (coverFit). */
+  cover: { sx: number; sy: number; ox: number; oy: number };
+}
+
+/** Where the person's edge ramps, in mask confidence. */
+const EDGE: [number, number] = [0.35, 0.7];
+
+type Program = { program: WebGLProgram; uniforms: Map<string, WebGLUniformLocation | null> };
+
+export class EffectsRenderer {
+  readonly canvas: HTMLCanvasElement | OffscreenCanvas;
+  private readonly gl: WebGL2RenderingContext;
+  private readonly downsample: Program;
+  private readonly blur: Program;
+  private readonly composite: Program;
+  private readonly frameTexture: WebGLTexture;
+  private readonly maskTexture: WebGLTexture;
+  private readonly pictureTexture: WebGLTexture;
+  private readonly blurTextures: [WebGLTexture, WebGLTexture];
+  private readonly blurBuffers: [WebGLFramebuffer, WebGLFramebuffer];
+  private readonly vertices: WebGLBuffer;
+  private readonly floatTargets: boolean;
+  private width = 0;
+  private height = 0;
+  private blurWidth = 0;
+  private blurHeight = 0;
+  private maskWidth = 1;
+  private maskHeight = 1;
+  private pictureSize: { width: number; height: number } | null = null;
+
+  constructor(canvas: HTMLCanvasElement | OffscreenCanvas) {
+    this.canvas = canvas;
+    const gl = canvas.getContext('webgl2', { alpha: false, antialias: false, depth: false, stencil: false, premultipliedAlpha: false, desynchronized: true }) as WebGL2RenderingContext | null;
+    if (!gl) throw new Error('WebGL2 is not available');
+    this.gl = gl;
+    this.floatTargets = gl.getExtension('EXT_color_buffer_float') !== null || gl.getExtension('EXT_color_buffer_half_float') !== null;
+    this.downsample = this.program(DOWNSAMPLE, ['frame', 'mask', 'maskReady', 'edge', 'flipY']);
+    this.blur = this.program(BLUR, ['source', 'direction', 'sigma', 'flipY']);
+    this.composite = this.program(COMPOSITE, ['frame', 'mask', 'blurred', 'picture', 'mode', 'maskReady', 'edge', 'texel', 'maskTexel', 'cover', 'beauty', 'beautyOn', 'beautyRadius', 'flipY']);
+    this.vertices = gl.createBuffer()!;
+    gl.bindBuffer(gl.ARRAY_BUFFER, this.vertices);
+    // One triangle covering the viewport.
+    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 3, -1, -1, 3]), gl.STATIC_DRAW);
+    this.frameTexture = this.texture();
+    this.maskTexture = this.texture();
+    this.pictureTexture = this.texture();
+    this.blurTextures = [this.texture(), this.texture()];
+    this.blurBuffers = [gl.createFramebuffer()!, gl.createFramebuffer()!];
+    // Until a mask arrives: one texel, person nowhere.
+    gl.bindTexture(gl.TEXTURE_2D, this.maskTexture);
+    gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, 1, 1, 0, gl.RED, gl.UNSIGNED_BYTE, new Uint8Array([0]));
+    gl.bindTexture(gl.TEXTURE_2D, this.pictureTexture);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0, 255]));
+  }
+
+  get lost(): boolean {
+    return this.gl.isContextLost();
+  }
+
+  /** The picture's size; the canvas and the blur follow it. */
+  setSize(width: number, height: number): void {
+    if (width === this.width && height === this.height) return;
+    this.width = width;
+    this.height = height;
+    this.canvas.width = width;
+    this.canvas.height = height;
+    this.blurWidth = Math.max(1, Math.round(width / 4));
+    this.blurHeight = Math.max(1, Math.round(height / 4));
+    const gl = this.gl;
+    for (let i = 0; i < 2; i += 1) {
+      gl.bindTexture(gl.TEXTURE_2D, this.blurTextures[i]);
+      if (this.floatTargets) gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA16F, this.blurWidth, this.blurHeight, 0, gl.RGBA, gl.HALF_FLOAT, null);
+      else gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.blurWidth, this.blurHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+      gl.bindFramebuffer(gl.FRAMEBUFFER, this.blurBuffers[i]);
+      gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this.blurTextures[i], 0);
+    }
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+  }
+
+  get size(): { width: number; height: number; blurHeight: number } {
+    return { width: this.width, height: this.height, blurHeight: this.blurHeight };
+  }
+
+  /** The camera frame (a VideoFrame or a playing video). */
+  uploadFrame(source: TexImageSource): void {
+    const gl = this.gl;
+    gl.bindTexture(gl.TEXTURE_2D, this.frameTexture);
+    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
+  }
+
+  /** The person mask, one byte per pixel, picture order. */
+  uploadMask(data: Uint8Array, width: number, height: number): void {
+    const gl = this.gl;
+    gl.bindTexture(gl.TEXTURE_2D, this.maskTexture);
+    gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, width, height, 0, gl.RED, gl.UNSIGNED_BYTE, data);
+    this.maskWidth = width;
+    this.maskHeight = height;
+  }
+
+  /** The background picture, or null to let it go. */
+  setPicture(image: TexImageSource | null, width = 1, height = 1): void {
+    const gl = this.gl;
+    gl.bindTexture(gl.TEXTURE_2D, this.pictureTexture);
+    if (image) {
+      gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
+      this.pictureSize = { width, height };
+    } else {
+      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0, 255]));
+      this.pictureSize = null;
+    }
+  }
+
+  get picture(): { width: number; height: number } | null {
+    return this.pictureSize;
+  }
+
+  /** Draw the finished picture onto the canvas. */
+  render(settings: RenderSettings): void {
+    const gl = this.gl;
+    if (gl.isContextLost() || this.width === 0) return;
+    gl.disable(gl.BLEND);
+    gl.disable(gl.DEPTH_TEST);
+    if (settings.mode === 'blur') {
+      gl.viewport(0, 0, this.blurWidth, this.blurHeight);
+      this.pass(this.downsample, this.blurBuffers[0], 0, (u) => {
+        this.bindTexture(0, this.frameTexture, u.get('frame'));
+        this.bindTexture(1, this.maskTexture, u.get('mask'));
+        gl.uniform1f(u.get('maskReady')!, settings.maskReady ? 1 : 0);
+        gl.uniform2f(u.get('edge')!, EDGE[0], EDGE[1]);
+      });
+      const sigma = Math.min(settings.blurSigma, 10.5);
+      this.pass(this.blur, this.blurBuffers[1], 0, (u) => {
+        this.bindTexture(0, this.blurTextures[0], u.get('source'));
+        gl.uniform2f(u.get('direction')!, 1 / this.blurWidth, 0);
+        gl.uniform1f(u.get('sigma')!, sigma);
+      });
+      this.pass(this.blur, this.blurBuffers[0], 0, (u) => {
+        this.bindTexture(0, this.blurTextures[1], u.get('source'));
+        gl.uniform2f(u.get('direction')!, 0, 1 / this.blurHeight);
+        gl.uniform1f(u.get('sigma')!, sigma);
+      });
+    }
+    gl.viewport(0, 0, this.width, this.height);
+    this.pass(this.composite, null, 1, (u) => {
+      this.bindTexture(0, this.frameTexture, u.get('frame'));
+      this.bindTexture(1, this.maskTexture, u.get('mask'));
+      this.bindTexture(2, this.blurTextures[0], u.get('blurred'));
+      this.bindTexture(3, this.pictureTexture, u.get('picture'));
+      gl.uniform1i(u.get('mode')!, settings.mode === 'blur' ? 1 : settings.mode === 'image' ? 2 : 0);
+      gl.uniform1f(u.get('maskReady')!, settings.maskReady ? 1 : 0);
+      gl.uniform2f(u.get('edge')!, EDGE[0], EDGE[1]);
+      gl.uniform2f(u.get('texel')!, 1 / this.width, 1 / this.height);
+      gl.uniform2f(u.get('maskTexel')!, 1 / this.maskWidth, 1 / this.maskHeight);
+      gl.uniform4f(u.get('cover')!, settings.cover.sx, settings.cover.sy, settings.cover.ox, settings.cover.oy);
+      const b = settings.beauty;
+      gl.uniform4f(u.get('beauty')!, b?.smooth ?? 0, b?.brighten ?? 0, b?.redness ?? 0, b?.sharpen ?? 0);
+      gl.uniform1f(u.get('beautyOn')!, b ? 1 : 0);
+      // About 3 px at 720p: the skin's pores and blemishes, not its features.
+      gl.uniform1f(u.get('beautyRadius')!, Math.max(1.5, this.height / 240));
+    });
+  }
+
+  /** Let the GPU resources go. */
+  dispose(): void {
+    const gl = this.gl;
+    for (const t of [this.frameTexture, this.maskTexture, this.pictureTexture, ...this.blurTextures]) gl.deleteTexture(t);
+    for (const f of this.blurBuffers) gl.deleteFramebuffer(f);
+    gl.deleteBuffer(this.vertices);
+    for (const p of [this.downsample, this.blur, this.composite]) gl.deleteProgram(p.program);
+    gl.getExtension('WEBGL_lose_context')?.loseContext();
+  }
+
+  private pass(p: Program, target: WebGLFramebuffer | null, flipY: number, setup: (u: Map<string, WebGLUniformLocation | null>) => void): void {
+    const gl = this.gl;
+    gl.bindFramebuffer(gl.FRAMEBUFFER, target);
+    gl.useProgram(p.program);
+    gl.uniform1f(p.uniforms.get('flipY')!, flipY);
+    setup(p.uniforms);
+    const at = gl.getAttribLocation(p.program, 'position');
+    gl.bindBuffer(gl.ARRAY_BUFFER, this.vertices);
+    gl.enableVertexAttribArray(at);
+    gl.vertexAttribPointer(at, 2, gl.FLOAT, false, 0, 0);
+    gl.drawArrays(gl.TRIANGLES, 0, 3);
+  }
+
+  private bindTexture(unit: number, texture: WebGLTexture, location: WebGLUniformLocation | null | undefined): void {
+    const gl = this.gl;
+    gl.activeTexture(gl.TEXTURE0 + unit);
+    gl.bindTexture(gl.TEXTURE_2D, texture);
+    if (location) gl.uniform1i(location, unit);
+  }
+
+  private texture(): WebGLTexture {
+    const gl = this.gl;
+    const t = gl.createTexture()!;
+    gl.bindTexture(gl.TEXTURE_2D, t);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    return t;
+  }
+
+  private program(fragment: string, names: string[]): Program {
+    const gl = this.gl;
+    const compile = (type: number, source: string): WebGLShader => {
+      const shader = gl.createShader(type)!;
+      gl.shaderSource(shader, source);
+      gl.compileShader(shader);
+      if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS) && !gl.isContextLost()) {
+        throw new Error(`video effects: a shader did not compile: ${gl.getShaderInfoLog(shader)}`);
+      }
+      return shader;
+    };
+    const program = gl.createProgram()!;
+    gl.attachShader(program, compile(gl.VERTEX_SHADER, VERTEX));
+    gl.attachShader(program, compile(gl.FRAGMENT_SHADER, fragment));
+    gl.linkProgram(program);
+    if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) {
+      throw new Error(`video effects: a shader did not link: ${gl.getProgramInfoLog(program)}`);
+    }
+    return { program, uniforms: new Map(names.map((n) => [n, gl.getUniformLocation(program, n)])) };
+  }
+}

package/src/fallback.ts

@@ -0,0 +1,169 @@
+/**
+ * Audio-only fallback (docs/implementation-progress/RTC_PERF_PROGRESS.md H1).
+ *
+ * When this viewer's connection is so poor that even the sound breaks up, remote camera videos are paused at the
+ * server so what bandwidth there is goes to the audio, and they come back once the connection has recovered.
+ *
+ * - Agora's `setStreamFallbackOption(uid, AUDIO_ONLY)` drops to the low stream and then to audio only, and reports
+ *   `stream-fallback` (fallback / recover).
+ * - Amazon Chime watches the audio it receives (`UnusableAudioWarningConnectionHealthPolicy`): about half of the
+ *   expected packets missing over 15 one-second samples suggests stopping video, with a 60 s cool-down.
+ *
+ * This follows Chime's signal -- the audio actually received -- measured as RTP loss (sequence gaps) rather than
+ * packet counts, because Opus DTX sends almost nothing during silence and a count would read silence as loss.
+ * Extreme only: {@link FALLBACK_LOSS} of the audio lost over {@link FALLBACK_SAMPLES} samples, with at least
+ * {@link MIN_PACKETS} packets to judge by. Back after a hold ({@link FIRST_HOLD_MS}, doubled for each relapse within
+ * {@link RELAPSE_MS}, up to {@link MAX_HOLD_MS}) once the loss is under {@link RECOVER_LOSS} -- or nobody spoke
+ * enough to tell, in which case video comes back and a still-bad connection falls back again, holding longer.
+ *
+ * The SFU cutting a video for bandwidth by itself is reported separately, by the server (`reason: 'bandwidth'`).
+ * Pure: the decisions take counters and a clock.
+ */
+import type { StatsEntry } from './stats.js';
+
+/** How often the received audio is sampled while the fallback could be needed. */
+export const PROTECT_SAMPLE_MS = 3000;
+/** This share of the audio lost (RTP sequence gaps) over the window: fall back. */
+export const FALLBACK_LOSS = 0.25;
+/** Below this share lost over the recovery window: video may come back. */
+export const RECOVER_LOSS = 0.05;
+/** Samples judged for falling back (15 s at 3 s) -- at least 3 of them. */
+export const FALLBACK_SAMPLES = 5;
+/** Samples judged for coming back (12 s). */
+export const RECOVER_SAMPLES = 4;
+/** Fewer audio packets than this over the fallback window are too few to judge. */
+export const MIN_PACKETS = 150;
+/** Fewer than this over the recovery window: nobody spoke enough to tell. */
+export const MIN_RECOVER_PACKETS = 30;
+/** The first fallback lasts at least this long. */
+export const FIRST_HOLD_MS = 20_000;
+/** A relapse doubles the hold, up to this. */
+export const MAX_HOLD_MS = 160_000;
+/** Falling back again within this long of coming back is a relapse. */
+export const RELAPSE_MS = 120_000;
+
+export interface AudioCounters {
+  received: number;
+  lost: number;
+}
+
+const num = (value: unknown): number => (typeof value === 'number' && Number.isFinite(value) ? value : 0);
+
+/** The received-audio counters of a receive transport's report, summed over its audio streams. */
+export function readInboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let received = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null || e.type !== 'inbound-rtp') continue;
+    if (e.kind !== 'audio' && e.mediaType !== 'audio') continue;
+    received += num(e.packetsReceived);
+    lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received, lost };
+}
+
+/**
+ * The sent-audio counters of a send transport's report: packets sent, and how many of them the server reported
+ * lost (its RTCP receiver reports). For the publisher side (Agora's setLocalPublishFallbackOption): `received` is what
+ * got through.
+ */
+export function readOutboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let sent = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null) continue;
+    const audio = e.kind === 'audio' || e.mediaType === 'audio';
+    if (!audio) continue;
+    if (e.type === 'outbound-rtp') sent += num(e.packetsSent);
+    else if (e.type === 'remote-inbound-rtp') lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received: Math.max(0, sent - lost), lost };
+}
+
+/** Decides when to fall back to audio only and when video may come back. */
+export class AudioProtection {
+  private samples: AudioCounters[] = [];
+  private previous: AudioCounters | null = null;
+  private activeSince: number | null = null;
+  private hold = FIRST_HOLD_MS;
+  private recoveredAt: number | null = null;
+
+  constructor(
+    private readonly onChange: (active: boolean) => void,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  /** Whether remote camera videos are held back now. */
+  get active(): boolean {
+    return this.activeSince !== null;
+  }
+
+  /** How long the current (or next first) fallback holds, in ms. */
+  get holdMs(): number {
+    return this.hold;
+  }
+
+  /**
+   * One reading of the cumulative counters. They only go up on one transport; lower ones are a new transport, and
+   * judging starts over from them.
+   */
+  feed(counters: AudioCounters): void {
+    const before = this.previous;
+    this.previous = counters;
+    if (before === null || counters.received < before.received || counters.lost < before.lost) {
+      this.samples = [];
+      return;
+    }
+    this.samples.push({ received: counters.received - before.received, lost: counters.lost - before.lost });
+    if (this.samples.length > Math.max(FALLBACK_SAMPLES, RECOVER_SAMPLES)) this.samples.shift();
+
+    if (this.activeSince === null) {
+      const window = this.samples.slice(-FALLBACK_SAMPLES);
+      if (window.length < 3) return;
+      const { total, lost } = sum(window);
+      if (total >= MIN_PACKETS && lost / total >= FALLBACK_LOSS) this.activate();
+      return;
+    }
+    if (this.now() - this.activeSince < this.hold || this.samples.length < RECOVER_SAMPLES) return;
+    const { total, lost } = sum(this.samples.slice(-RECOVER_SAMPLES));
+    if (total < MIN_RECOVER_PACKETS || lost / total < RECOVER_LOSS) this.recover();
+  }
+
+  /** Come back now (the app turned the fallback off). */
+  release(): void {
+    if (this.activeSince !== null) this.recover();
+  }
+
+  /** The transports went away (reconnect, close): forget everything, without telling. */
+  reset(): void {
+    this.activeSince = null;
+    this.samples = [];
+    this.previous = null;
+  }
+
+  private activate(): void {
+    const now = this.now();
+    const relapse = this.recoveredAt !== null && now - this.recoveredAt < RELAPSE_MS;
+    this.hold = relapse ? Math.min(this.hold * 2, MAX_HOLD_MS) : FIRST_HOLD_MS;
+    this.activeSince = now;
+    this.samples = [];
+    this.onChange(true);
+  }
+
+  private recover(): void {
+    this.activeSince = null;
+    this.recoveredAt = this.now();
+    this.samples = [];
+    this.onChange(false);
+  }
+}
+
+function sum(samples: readonly AudioCounters[]): { total: number; lost: number } {
+  let total = 0;
+  let lost = 0;
+  for (const s of samples) {
+    total += s.received + s.lost;
+    lost += s.lost;
+  }
+  return { total, lost };
+}

package/src/health.ts

@@ -0,0 +1,127 @@
+/**
+ * Media health warnings (docs/implementation-progress/RTC_PERF_PROGRESS.md K1).
+ *
+ * - Agora's `exception` event pairs each problem with its recovery: FRAMERATE_INPUT_TOO_LOW / _RECOVER (the camera
+ *   delivers too few frames), AUDIO_INPUT_LEVEL_TOO_LOW / _RECOVER, ...
+ * - LiveKit's `AudioSilenceDetected`: a microphone that delivers nothing but silence (a hardware mute switch, a
+ *   broken device) right after it starts.
+ * - LiveKit watches the track's `mute` event (the OS took the device: a phone call, another app) and reacts after 5 s.
+ *
+ * Each warning is reported once when it starts (`active: true`) and once when it clears (`active: false`).
+ * Pure: the decisions take samples.
+ */
+
+export type MediaWarningCode =
+  /** The microphone is on but delivers exact digital silence: a mute switch, a broken or wrongly chosen device. */
+  | 'mic-silent'
+  /** The camera delivers far fewer frames than asked: usually a dark room (the camera exposes longer), or a busy device. */
+  | 'camera-fps-low'
+  /** The system took the camera from this page for now (a phone call, another app, a locked screen). */
+  | 'camera-interrupted'
+  /** The system took the microphone (a phone call, another app). */
+  | 'mic-interrupted';
+
+export interface MediaWarning {
+  code: MediaWarningCode;
+  /** true when it starts, false when it clears. */
+  active: boolean;
+}
+
+/** How often the published camera and microphone are sampled while someone listens. */
+export const HEALTH_SAMPLE_MS = 2000;
+/** Samples of exact silence before mic-silent (6 s). */
+export const SILENT_SAMPLES = 3;
+/** Below this the level counts as exact digital silence (a real room's noise floor is well above it). */
+export const SILENCE_LEVEL = 0.0001;
+/** Fewer frames than this per second, when 24 or more were asked, is camera-fps-low. */
+export const LOW_FPS = 15;
+/** Samples below LOW_FPS before warning (6 s); samples at RECOVERED_FPS or more before clearing. */
+export const LOW_FPS_SAMPLES = 3;
+export const RECOVERED_FPS = 20;
+export const RECOVER_SAMPLES = 2;
+/** A system mute shorter than this is a glitch, not an interruption. */
+export const INTERRUPTED_AFTER_MS = 2000;
+
+/** One reading of the published devices. null: not published (or nothing measured). */
+export interface HealthSample {
+  /** The microphone's level, while it is on. */
+  micLevel: number | null;
+  /** The frames per second the camera delivers, while it is on, and what was asked. */
+  cameraFps: number | null;
+  askedFps: number;
+}
+
+/** Decides mic-silent and camera-fps-low from samples. */
+export class HealthDecisions {
+  private silent = 0;
+  private lowFps = 0;
+  private goodFps = 0;
+  private readonly activeNow = new Set<MediaWarningCode>();
+
+  constructor(private readonly report: (warning: MediaWarning) => void) {}
+
+  isActive(code: MediaWarningCode): boolean {
+    return this.activeNow.has(code);
+  }
+
+  feed(sample: HealthSample): void {
+    // mic-silent: exact silence held; any sound clears it. Off or unpublished: not a warning.
+    if (sample.micLevel === null) {
+      this.silent = 0;
+      this.set('mic-silent', false);
+    } else if (sample.micLevel <= SILENCE_LEVEL) {
+      this.silent += 1;
+      if (this.silent >= SILENT_SAMPLES) this.set('mic-silent', true);
+    } else {
+      this.silent = 0;
+      this.set('mic-silent', false);
+    }
+
+    // camera-fps-low: only when a smooth rate was asked; cleared once it holds again.
+    if (sample.cameraFps === null || sample.askedFps < 24) {
+      this.lowFps = 0;
+      this.goodFps = 0;
+      this.set('camera-fps-low', false);
+    } else if (sample.cameraFps < LOW_FPS) {
+      this.goodFps = 0;
+      this.lowFps += 1;
+      if (this.lowFps >= LOW_FPS_SAMPLES) this.set('camera-fps-low', true);
+    } else {
+      this.lowFps = 0;
+      if (sample.cameraFps >= RECOVERED_FPS) this.goodFps += 1;
+      if (this.goodFps >= RECOVER_SAMPLES) this.set('camera-fps-low', false);
+    }
+  }
+
+  /** A system interruption started or ended (decided by the caller's timer). */
+  interrupted(code: 'camera-interrupted' | 'mic-interrupted', active: boolean): void {
+    this.set(code, active);
+  }
+
+  /** Everything clears (unpublished, closed): each active warning is reported cleared. */
+  clearAll(): void {
+    for (const code of [...this.activeNow]) this.set(code, false);
+    this.silent = 0;
+    this.lowFps = 0;
+    this.goodFps = 0;
+  }
+
+  private set(code: MediaWarningCode, active: boolean): void {
+    if (this.activeNow.has(code) === active) return;
+    if (active) this.activeNow.add(code);
+    else this.activeNow.delete(code);
+    this.report({ code, active });
+  }
+}
+
+/** The microphone level and camera frame rate from a producer's statistics (media-source). */
+export function readMediaSource(report: Iterable<Record<string, unknown>>): { level: number | null; fps: number | null } {
+  let level: number | null = null;
+  let fps: number | null = null;
+  for (const s of report) {
+    if (s.type !== 'media-source') continue;
+    if (typeof s.audioLevel === 'number') level = s.audioLevel;
+    if (typeof s.framesPerSecond === 'number') fps = s.framesPerSecond;
+  }
+  return { level, fps };
+}