@voltkit-rtc/client 0.7.3 → 0.9.0

package/src/link.ts

@@ -0,0 +1,255 @@
+/**
+ * Keeping the connection alive between drops (docs/implementation-progress/RTC_PERF_PROGRESS.md G1, G2).
+ *
+ * Two failures the socket's own close never reports:
+ *
+ * - **The media path dies while the socket lives** (Wi-Fi to mobile data, a NAT rebinding, UDP blocked mid-call).
+ *   The video freezes for good. Twilio and LiveKit restart ICE on the same transports first -- new ICE credentials
+ *   from the server, the same producers and consumers -- and rejoin only when that does not bring the path back.
+ *   {@link MediaPathWatch} does the same: a transport `disconnected` for {@link DISCONNECTED_GRACE_MS} or `failed`
+ *   restarts ICE; not `connected` within {@link RESTART_TIMEOUT_MS} of starting recovery (including the request), or a server without `restartIce`, is a
+ *   full reconnect.
+ * - **The socket dies silently**: the network went away and TCP has not noticed yet, which can take minutes. The
+ *   server pings, but a browser cannot see those. LiveKit pings from the client and reconnects when no pong comes.
+ *   {@link SocketLiveness} sends `ping` after {@link IDLE_MS} with nothing received, and calls the socket dead when
+ *   nothing at all arrives within {@link PING_TIMEOUT_MS} of it. Any frame counts, so an older server's
+ *   `unknown-method` reply proves the socket as well as a pong.
+ *
+ * Pure: timers and the clock are injectable, so the decisions are unit-tested on their own.
+ */
+
+/** A transport `disconnected` this long is restarted (it often recovers by itself within a second or two). */
+export const DISCONNECTED_GRACE_MS = 3000;
+/** The whole ICE recovery, including the request, has this long before the session rejoins. */
+export const RESTART_TIMEOUT_MS = 10_000;
+/** With nothing received for this long, the client pings. */
+export const IDLE_MS = 15_000;
+/** A ping with nothing at all received for this long after it: the socket is dead. */
+export const PING_TIMEOUT_MS = 8000;
+/** How often liveness is checked. Background tabs may run it less often; the decisions use the clock, not counts. */
+export const LIVENESS_TICK_MS = 5000;
+
+export type TransportState = 'new' | 'connecting' | 'connected' | 'disconnected' | 'failed' | 'closed';
+
+/** What an ICE restart came to. */
+export type RestartOutcome =
+  /** New credentials applied: wait for `connected`. */
+  | 'restarted'
+  /** The server has no restartIce (older than 2026-10-19): rejoin instead. */
+  | 'unsupported'
+  /** It could not be done now (the socket is down, the session is going): the socket's own recovery handles it. */
+  | 'skipped';
+
+export interface MediaPathHost {
+  /** Whether the session is connected; nothing is done otherwise (a reconnect is already under way, or it ended). */
+  active(): boolean;
+  /** Ask the server for new ICE credentials for this transport and apply them. */
+  restartIce(transportId: string): Promise<RestartOutcome>;
+  /** Give up on this connection and rejoin. */
+  reconnect(reason: string): void;
+  /** Told when a transport's media path is lost and when it is back (for `mediaReconnecting`-style UI). */
+  onPathChange?(transportId: string, lost: boolean): void;
+  setTimer?(fn: () => void, ms: number): unknown;
+  clearTimer?(handle: unknown): void;
+}
+
+interface PathEntry {
+  revision: number;
+  state: TransportState;
+  timer: unknown;
+  /** One restart is in flight; repeated loss notifications share its deadline. */
+  restarting: boolean;
+  lost: boolean;
+}
+
+/** Watches each media transport's connection state and recovers a dead path. */
+export class MediaPathWatch {
+  private readonly entries = new Map<string, PathEntry>();
+  private readonly setTimer: (fn: () => void, ms: number) => unknown;
+  private readonly clearTimer: (handle: unknown) => void;
+
+  constructor(private readonly host: MediaPathHost) {
+    this.setTimer = host.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+    this.clearTimer = host.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>));
+  }
+
+  /** A transport's connection state changed (mediasoup-client `connectionstatechange`). */
+  update(transportId: string, state: TransportState): void {
+    let entry = this.entries.get(transportId);
+    if (!entry) {
+      entry = { state, timer: null, restarting: false, lost: false, revision: 0 };
+      this.entries.set(transportId, entry);
+    }
+    entry.state = state;
+    switch (state) {
+      case 'connected':
+        entry.revision++;
+        this.cancel(entry);
+        entry.restarting = false;
+        this.setLost(transportId, entry, false);
+        return;
+      case 'closed':
+        this.forget(transportId);
+        return;
+      case 'disconnected':
+        // A restart is already waiting for `connected`; its own deadline decides.
+        if (entry.restarting || entry.timer !== null) return;
+        entry.timer = this.setTimer(() => {
+          entry.timer = null;
+          if (entry.state === 'disconnected' || entry.state === 'failed') void this.restart(transportId, entry);
+        }, DISCONNECTED_GRACE_MS);
+        return;
+      case 'failed':
+        if (entry.restarting) return; // duplicate notification; the deadline decides
+        this.cancel(entry);
+        void this.restart(transportId, entry);
+        return;
+      default:
+        // new / connecting: a fresh transport, or one mid-restart.
+        return;
+    }
+  }
+
+  /** The transport went away with its session (teardown): no more decisions for it. */
+  forget(transportId: string): void {
+    const entry = this.entries.get(transportId);
+    if (!entry) return;
+    this.cancel(entry);
+    this.entries.delete(transportId);
+    if (entry.lost) this.host.onPathChange?.(transportId, false);
+  }
+
+  /** Every transport went away (teardown, close). */
+  reset(): void {
+    for (const id of [...this.entries.keys()]) this.forget(id);
+  }
+
+  /** For tests and the network-change nudge: is this transport being recovered now? */
+  restartingNow(transportId: string): boolean {
+    return this.entries.get(transportId)?.restarting === true;
+  }
+
+  private async restart(transportId: string, entry: PathEntry): Promise<void> {
+    if (!this.host.active() || this.entries.get(transportId) !== entry) return;
+    entry.restarting = true;
+    const revision = ++entry.revision;
+    // Include both the RPC and the platform call in the deadline: neither is
+    // guaranteed to settle when a network/interface disappears.
+    this.cancel(entry);
+    entry.timer = this.setTimer(() => {
+      entry.timer = null;
+      if (!entry.restarting || this.entries.get(transportId) !== entry) return;
+      entry.revision++;
+      entry.restarting = false;
+      if (this.host.active()) this.host.reconnect('media-timeout');
+    }, RESTART_TIMEOUT_MS);
+    this.setLost(transportId, entry, true);
+    let outcome: RestartOutcome;
+    try {
+      outcome = await this.host.restartIce(transportId);
+    } catch {
+      outcome = 'unsupported';
+    }
+    if (this.entries.get(transportId) !== entry || entry.revision !== revision) return;
+    if (outcome === 'skipped') {
+      this.cancel(entry);
+      entry.restarting = false;
+      return;
+    }
+    if (outcome === 'unsupported') {
+      this.cancel(entry);
+      entry.restarting = false;
+      if (this.host.active()) this.host.reconnect('media-failed');
+      return;
+    }
+    // `connected` clears the deadline, including when it arrives during the await.
+  }
+
+  private setLost(transportId: string, entry: PathEntry, lost: boolean): void {
+    if (entry.lost === lost) return;
+    entry.lost = lost;
+    this.host.onPathChange?.(transportId, lost);
+  }
+
+  private cancel(entry: PathEntry): void {
+    if (entry.timer !== null) this.clearTimer(entry.timer);
+    entry.timer = null;
+  }
+}
+
+export interface LivenessHost {
+  /** Whether the session's socket should be checked now (connected, not closing). */
+  active(): boolean;
+  /** Send a ping; its reply (or error reply) reaches {@link SocketLiveness.received} like any frame. */
+  ping(): void;
+  /** Nothing came back: close the socket so the normal reconnect runs. */
+  dead(): void;
+  now?(): number;
+  setInterval?(fn: () => void, ms: number): unknown;
+  clearInterval?(handle: unknown): void;
+}
+
+/** Notices a socket that died without closing. */
+export class SocketLiveness {
+  private lastReceived = 0;
+  private pingSentAt: number | null = null;
+  private timer: unknown = null;
+  private readonly now: () => number;
+
+  constructor(private readonly host: LivenessHost) {
+    this.now = host.now ?? Date.now;
+  }
+
+  /** Start watching a newly opened socket. */
+  start(): void {
+    this.stop();
+    this.lastReceived = this.now();
+    this.pingSentAt = null;
+    const every = this.host.setInterval ?? ((fn, ms) => setInterval(fn, ms));
+    this.timer = every(() => this.check(), LIVENESS_TICK_MS);
+    // A Node/SSR process with no other work need not stay alive for a browser
+    // connection watch. Browser timers are numbers and have no unref.
+    (this.timer as { unref?: () => void })?.unref?.();
+  }
+
+  stop(): void {
+    if (this.timer !== null) (this.host.clearInterval ?? ((h) => clearInterval(h as ReturnType<typeof setInterval>)))(this.timer);
+    this.timer = null;
+    this.pingSentAt = null;
+  }
+
+  get running(): boolean {
+    return this.timer !== null;
+  }
+
+  /** Any frame arrived on the socket. */
+  received(): void {
+    this.lastReceived = this.now();
+    this.pingSentAt = null;
+  }
+
+  /** Check now instead of at the next tick (the browser said the network changed). */
+  checkNow(): void {
+    if (this.timer === null || !this.host.active()) return;
+    if (this.pingSentAt === null) {
+      this.pingSentAt = this.now();
+      this.host.ping();
+    }
+  }
+
+  check(): void {
+    if (this.timer === null || !this.host.active()) return;
+    const now = this.now();
+    if (this.pingSentAt !== null) {
+      if (now - this.pingSentAt >= PING_TIMEOUT_MS) {
+        this.stop();
+        this.host.dead();
+      }
+      return;
+    }
+    if (now - this.lastReceived >= IDLE_MS) {
+      this.pingSentAt = now;
+      this.host.ping();
+    }
+  }
+}

package/src/noise-suppression.ts

@@ -0,0 +1,135 @@
+/**
+ * AI noise suppression for the microphone (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-4): what Agora's AI
+ * Denoiser, LiveKit's Krisp filter and ZEGOCLOUD's AI noise suppression do, with RNNoise -- the open neural-network
+ * denoiser Jitsi uses -- through `@sapphi-red/web-noise-suppressor` (MIT): a Web Audio worklet in the browser's
+ * audio thread. Fans, traffic and room rumble fall away (measured in Chromium: rumble-like noise down by more than
+ * 60 dB, steady white hiss by 7-12 dB); voices stay.
+ *
+ * Imported on its own (`@voltkit-rtc/client/noise-suppression`), so an app that does not use it never ships it:
+ *
+ *   import { createNoiseSuppressor } from '@voltkit-rtc/client/noise-suppression';
+ *   await client.setAudioProcessor(createNoiseSuppressor());
+ *
+ * The worklet (64 kB) and the model (150 kB) are fetched when it starts: from jsDelivr, at the exact version this
+ * SDK was tested with, unless you serve them yourself (`workletUrl`, `wasmUrl`, `simdWasmUrl`) -- copy them from
+ * `node_modules/@sapphi-red/web-noise-suppressor/dist` to your own site for a strict Content-Security-Policy.
+ */
+import type { AudioTrackProcessor } from './processors.js';
+
+/** The package version the default URLs point at; equal to this SDK's pinned dependency (a test checks it). */
+export const NOISE_SUPPRESSOR_VERSION = '0.4.1';
+const CDN = `https://cdn.jsdelivr.net/npm/@sapphi-red/web-noise-suppressor@${NOISE_SUPPRESSOR_VERSION}/dist`;
+
+export interface NoiseSuppressorOptions {
+  /** Where `rnnoise/workletProcessor.js` is served. Default: jsDelivr, pinned. */
+  workletUrl?: string;
+  /** Where `rnnoise.wasm` is served. Default: jsDelivr, pinned. */
+  wasmUrl?: string;
+  /** Where `rnnoise_simd.wasm` is served (used where the browser supports SIMD). Default: jsDelivr, pinned. */
+  simdWasmUrl?: string;
+}
+
+/** What the processor needs from the browser; stood in for by tests. */
+export interface NoiseSuppressorDeps {
+  createContext(): AudioContext;
+  loadModel(urls: { url: string; simdUrl: string }): Promise<ArrayBuffer>;
+  createNode(context: AudioContext, wasmBinary: ArrayBuffer): Promise<DenoiseNode> | DenoiseNode;
+  createStream(tracks: MediaStreamTrack[]): MediaStream;
+}
+
+type DenoiseNode = AudioNode & { destroy?(): void };
+type Library = typeof import('@sapphi-red/web-noise-suppressor');
+let library: Promise<Library> | null = null;
+
+/**
+ * The library is loaded on first use, not with this module: it extends `AudioWorkletNode` as it loads, which exists
+ * only in browsers, so a static import would break a server render (Next.js) or Node.
+ */
+const load = (): Promise<Library> =>
+  (library ??= import('@sapphi-red/web-noise-suppressor').catch((err: unknown) => {
+    library = null;
+    throw err;
+  }));
+
+const browserDeps: NoiseSuppressorDeps = {
+  // RNNoise works on 48 kHz audio.
+  createContext: () => new AudioContext({ sampleRate: 48000 }),
+  loadModel: async (urls) => (await load()).loadRnnoise(urls),
+  createNode: async (context, wasmBinary) => new (await load()).RnnoiseWorkletNode(context, { wasmBinary, maxChannels: 1 }),
+  createStream: (tracks) => new MediaStream(tracks),
+};
+
+/** An `AudioTrackProcessor` for `client.setAudioProcessor()` that removes background noise with RNNoise. */
+export function createNoiseSuppressor(options: NoiseSuppressorOptions = {}, deps: NoiseSuppressorDeps = browserDeps): AudioTrackProcessor {
+  let context: AudioContext | null = null;
+  let ready: Promise<ArrayBuffer> | null = null;
+  let graph: { source: AudioNode; node: DenoiseNode; output: MediaStreamTrack } | null = null;
+  let resumeOnGesture: (() => void) | null = null;
+
+  const release = () => {
+    if (!graph) return;
+    try {
+      graph.source.disconnect();
+      graph.node.disconnect();
+      graph.node.destroy?.();
+    } catch {
+      // Already gone.
+    }
+    graph.output.stop();
+    graph = null;
+  };
+
+  /** A context made without a tap starts suspended: no sound would go out until the page is touched. */
+  const keepRunning = (ctx: AudioContext) => {
+    void ctx.resume?.().catch(() => undefined);
+    if (ctx.state === 'running' || typeof document === 'undefined' || resumeOnGesture) return;
+    resumeOnGesture = () => {
+      void ctx.resume().catch(() => undefined);
+      if (ctx.state === 'running' && resumeOnGesture) {
+        for (const type of ['pointerdown', 'keydown'] as const) document.removeEventListener(type, resumeOnGesture, { capture: true });
+        resumeOnGesture = null;
+      }
+    };
+    for (const type of ['pointerdown', 'keydown'] as const) document.addEventListener(type, resumeOnGesture, { capture: true });
+  };
+
+  return {
+    name: 'noise-suppression',
+    async process(track: MediaStreamTrack): Promise<MediaStreamTrack> {
+      const ctx = (context ??= deps.createContext());
+      ready ??= Promise.all([
+        deps.loadModel({ url: options.wasmUrl ?? `${CDN}/rnnoise.wasm`, simdUrl: options.simdWasmUrl ?? `${CDN}/rnnoise_simd.wasm` }),
+        ctx.audioWorklet.addModule(options.workletUrl ?? `${CDN}/rnnoise/workletProcessor.js`),
+      ]).then(([model]) => model);
+      let model: ArrayBuffer;
+      try {
+        model = await ready;
+      } catch (err) {
+        ready = null; // try again next time (a network blip)
+        throw err;
+      }
+      const node = await deps.createNode(ctx, model);
+      release();
+      const source = ctx.createMediaStreamSource(deps.createStream([track]));
+      const destination = ctx.createMediaStreamDestination();
+      source.connect(node);
+      node.connect(destination);
+      const output = destination.stream.getAudioTracks()[0];
+      if (!output) throw new Error('the audio graph gave no track');
+      graph = { source, node, output };
+      keepRunning(ctx);
+      return output;
+    },
+    async destroy(): Promise<void> {
+      release();
+      if (resumeOnGesture && typeof document !== 'undefined') {
+        for (const type of ['pointerdown', 'keydown'] as const) document.removeEventListener(type, resumeOnGesture, { capture: true });
+      }
+      resumeOnGesture = null;
+      const ctx = context;
+      context = null;
+      ready = null;
+      await ctx?.close().catch(() => undefined);
+    },
+  };
+}

package/src/preflight.ts

@@ -0,0 +1,191 @@
+/**
+ * Pre-call check (docs/implementation-progress/RTC_PERF_PROGRESS.md L1).
+ *
+ * - Twilio's `runPreflight` (read from its code) joins a test room, publishes synthetic audio and video and reports
+ *   timings, the connection path, round trip, jitter and loss.
+ * - LiveKit's `ConnectionCheck` runs separate checks -- WebSocket, WebRTC, TURN, publishing audio and video, UDP
+ *   against TCP -- each with messages and warnings.
+ * - Agora's `startLastmileProbeTest` measures the last mile before a call.
+ *
+ * This joins the room the token names (mint one for a test room nobody else is in: the session is real, and billed
+ * like any other for its few seconds), publishes what it is given -- or a tone and a picture it makes -- and reports
+ * what it saw. The orchestration takes a small target interface, so it is tested without a browser.
+ */
+import { networkQualityOf, readTransportStats, type NetworkQualityLevel, type TransportSample } from './network.js';
+import type { StatsEntry } from './stats.js';
+import type { MediaWarningCode } from './health.js';
+
+export interface PreflightOptions {
+  /** What to publish (cloned; your tracks are left running). Without it, a tone and a moving picture are made. */
+  media?: MediaStream;
+  /** How long media is measured once it flows. Default 5 s. */
+  durationMs?: number;
+}
+
+export type PreflightStepName = 'join' | 'media';
+
+export interface PreflightStep {
+  name: PreflightStepName;
+  ok: boolean;
+  /** How long it took, in ms. */
+  ms: number | null;
+  error?: string;
+}
+
+export interface PreflightReport {
+  /** Joined, and media reached the server. */
+  ok: boolean;
+  steps: PreflightStep[];
+  /** How media travelled: UDP is best; TCP or a relay means a firewall in the way (it works, with more delay). */
+  path: { protocol: 'udp' | 'tcp' | null; relayed: boolean } | null;
+  rttMs: number | null;
+  lossPct: number | null;
+  /** The upload the browser estimated it could use, in kbps. */
+  uploadKbps: number | null;
+  level: NetworkQualityLevel | null;
+  /** Camera and microphone problems seen during the test (see the mediaWarning event). */
+  warnings: MediaWarningCode[];
+}
+
+/** What a pre-call check needs from a client. */
+export interface PreflightTarget {
+  connect(): Promise<unknown>;
+  publish(stream: MediaStream): Promise<void>;
+  /** The send transport's connection state, or null before there is one. */
+  sendState(): string | null;
+  /** The send transport's statistics, or null without one. */
+  sendStats(): Promise<Iterable<StatsEntry> | null>;
+  onWarning(listener: (code: MediaWarningCode, active: boolean) => void): () => void;
+  close(): void;
+}
+
+export interface PreflightEnv {
+  now(): number;
+  sleep(ms: number): Promise<void>;
+}
+
+const realEnv: PreflightEnv = { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) };
+
+/** Media has this long to connect once published. */
+export const MEDIA_CONNECT_TIMEOUT_MS = 10_000;
+/** Statistics are read this often while measuring. */
+export const PREFLIGHT_SAMPLE_MS = 1000;
+
+/** The selected candidate pair's protocol, and whether either end is a relay. */
+export function connectionPath(report: Iterable<StatsEntry>): PreflightReport['path'] {
+  const entries = [...report];
+  const byId = new Map<string, StatsEntry>();
+  for (const e of entries) if (typeof e.id === 'string') byId.set(e.id, e);
+  let pair: StatsEntry | undefined;
+  for (const t of entries.filter((e) => e.type === 'transport')) {
+    if (typeof t.selectedCandidatePairId === 'string') pair = byId.get(t.selectedCandidatePairId);
+    if (pair) break;
+  }
+  pair ??= entries.find((e) => e.type === 'candidate-pair' && e.nominated === true && e.state === 'succeeded');
+  if (!pair) return null;
+  const local = typeof pair.localCandidateId === 'string' ? byId.get(pair.localCandidateId) : undefined;
+  const remote = typeof pair.remoteCandidateId === 'string' ? byId.get(pair.remoteCandidateId) : undefined;
+  const protocol = local?.protocol === 'udp' || local?.protocol === 'tcp' ? local.protocol : null;
+  const relayed = local?.candidateType === 'relay' || remote?.candidateType === 'relay';
+  return { protocol, relayed };
+}
+
+/** Run the check on `target`, closing it at the end whatever happens. */
+export async function runPreflight(target: PreflightTarget, stream: MediaStream, options: PreflightOptions = {}, env: PreflightEnv = realEnv): Promise<PreflightReport> {
+  const report: PreflightReport = { ok: false, steps: [], path: null, rttMs: null, lossPct: null, uploadKbps: null, level: null, warnings: [] };
+  const seen = new Set<MediaWarningCode>();
+  const stopWarnings = target.onWarning((code, active) => { if (active) seen.add(code); });
+  try {
+    let started = env.now();
+    try {
+      await target.connect();
+      report.steps.push({ name: 'join', ok: true, ms: env.now() - started });
+    } catch (err) {
+      report.steps.push({ name: 'join', ok: false, ms: null, error: String((err as Error)?.message ?? err) });
+      return report;
+    }
+
+    started = env.now();
+    try {
+      await target.publish(stream);
+      while (target.sendState() !== 'connected') {
+        const state = target.sendState();
+        if (state === 'failed' || state === 'closed') throw new Error(`media connection ${state}`);
+        if (env.now() - started > MEDIA_CONNECT_TIMEOUT_MS) throw new Error('media did not connect in time (a firewall blocking UDP and TCP?)');
+        await env.sleep(100);
+      }
+      report.steps.push({ name: 'media', ok: true, ms: env.now() - started });
+    } catch (err) {
+      report.steps.push({ name: 'media', ok: false, ms: null, error: String((err as Error)?.message ?? err) });
+      return report;
+    }
+
+    // Measure: the worst round trip and loss over the run, the last upload estimate.
+    const duration = options.durationMs ?? 5000;
+    let previous = new Map<string, TransportSample>();
+    const until = env.now() + duration;
+    while (env.now() < until) {
+      await env.sleep(PREFLIGHT_SAMPLE_MS);
+      const stats = await target.sendStats().catch(() => null);
+      if (!stats) continue;
+      const entries = [...stats];
+      report.path = connectionPath(entries) ?? report.path;
+      const now = new Map([['send', readTransportStats(entries)]]);
+      const q = networkQualityOf(now, previous, true);
+      previous = now;
+      if (q.rttMs !== null) report.rttMs = Math.max(report.rttMs ?? 0, q.rttMs);
+      if (q.lossPct !== null) report.lossPct = Math.max(report.lossPct ?? 0, q.lossPct);
+      if (q.availableKbps !== null) report.uploadKbps = q.availableKbps;
+      report.level = q.level;
+    }
+    report.ok = true;
+    return report;
+  } finally {
+    report.warnings = [...seen];
+    stopWarnings();
+    target.close();
+  }
+}
+
+/**
+ * A tone and a moving picture to publish when the app gives none (Twilio's preflight makes synthetic media too).
+ * Returns the stream and a function that stops everything it made. Browser only.
+ */
+export function syntheticMedia(): { stream: MediaStream; stop: () => void } {
+  const tracks: MediaStreamTrack[] = [];
+  const stops: (() => void)[] = [];
+  const AudioCtx = (globalThis as { AudioContext?: typeof AudioContext }).AudioContext;
+  if (AudioCtx) {
+    const ctx = new AudioCtx();
+    const osc = ctx.createOscillator();
+    const gain = ctx.createGain();
+    gain.gain.value = 0.05;
+    const dest = ctx.createMediaStreamDestination();
+    osc.connect(gain).connect(dest);
+    osc.start();
+    tracks.push(...dest.stream.getAudioTracks());
+    stops.push(() => { osc.stop(); void ctx.close(); });
+  }
+  if (typeof document !== 'undefined') {
+    const canvas = document.createElement('canvas');
+    canvas.width = 640;
+    canvas.height = 360;
+    const g = canvas.getContext('2d');
+    let frame = 0;
+    const draw = () => {
+      if (!g) return;
+      frame += 1;
+      g.fillStyle = `hsl(${frame % 360}, 60%, 40%)`;
+      g.fillRect(0, 0, canvas.width, canvas.height);
+      g.fillStyle = '#fff';
+      g.fillRect((frame * 8) % canvas.width, 150, 60, 60);
+    };
+    draw();
+    const timer = setInterval(draw, 1000 / 30);
+    const captured = (canvas as HTMLCanvasElement & { captureStream?: (fps: number) => MediaStream }).captureStream?.(30);
+    if (captured) tracks.push(...captured.getVideoTracks());
+    stops.push(() => clearInterval(timer));
+  }
+  const stream = new MediaStream(tracks);
+  return { stream, stop: () => { for (const t of tracks) t.stop(); for (const s of stops) s(); } };
+}

package/src/processors.ts

@@ -0,0 +1,35 @@
+/**
+ * Track processors (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-4, P3-5): the published microphone or
+ * camera sent through something that changes it first -- noise suppression, a blurred or replaced background, beauty.
+ * LiveKit (`setProcessor`: Krisp, background blur), Agora and TRTC (extensions and plugins) do it the same way: the
+ * device's track goes in, a processed track goes out and is published in its place; the device's own track stays the
+ * one watched for unplugging, given new constraints and released at the end.
+ */
+
+/** Something the microphone passes through before it is sent. */
+export interface AudioTrackProcessor {
+  /** A short name, for diagnostics. */
+  readonly name: string;
+  /**
+   * Process `track` and return the track to send. Called again whenever the microphone's track changes (another
+   * device, a reconnect): the processor then lets go of what it made for the previous one.
+   */
+  process(track: MediaStreamTrack): Promise<MediaStreamTrack>;
+  /** Let go of everything (the processor was removed, or the session ended). */
+  destroy(): Promise<void> | void;
+}
+
+/** Something the camera passes through before it is sent (and shown in `client.local`). */
+export interface VideoTrackProcessor {
+  /** A short name, for diagnostics. */
+  readonly name: string;
+  /**
+   * Process `track` and return the track to send, at the same size and frame rate. Called again when the camera's
+   * track changes (another camera, `replaceVideoTrack`): the processor then lets go of what it made for the previous
+   * one. A picture the SDK no longer sends is stopped by the SDK; stop drawing it once its `readyState` is 'ended',
+   * and skip the work while it is switched off (`enabled` false: the camera is paused).
+   */
+  process(track: MediaStreamTrack): Promise<MediaStreamTrack>;
+  /** Let go of everything (the processor was removed, or the session ended). */
+  destroy(): Promise<void> | void;
+}

package/src/speakers.ts

@@ -0,0 +1,66 @@
+/**
+ * Audio levels and active speakers (docs/implementation-progress/RTC_PERF_PROGRESS.md J1).
+ *
+ * - Agora `volume-indicator` and Tencent TRTC `AUDIO_VOLUME` report every participant's volume on an interval.
+ * - Amazon Chime's `DefaultActiveSpeakerPolicy` (read from its code) ranks speakers: a volume above a silence
+ *   threshold counts as speaking (1, else 0); each sample the speaker's score becomes `score × 0.9 + speaking × 0.1`,
+ *   and everyone else's drops by `0.2 × speaking` (the takeover); scores under 0.01 are 0. Its detector updates every
+ *   200 ms. Sampled together here, so two people speaking at once both stay; the takeover applies to the silent.
+ * - LiveKit's server counts a participant as speaking above -35 dBov.
+ *
+ * Levels are WebRTC's `audioLevel`: linear, 0 = silence, 1 = 0 dBov. -35 dBov is 10^(-35/20) ≈ 0.0178.
+ * Pure: the ranking takes levels and gives the order.
+ */
+
+/** Above this level a participant is speaking (LiveKit's -35 dBov). */
+export const SPEAKING_LEVEL = 0.0178;
+/** How often levels are read while someone listens (Chime's detector: 200 ms). */
+export const AUDIO_LEVEL_INTERVAL_MS = 250;
+
+export const SPEAKER_WEIGHT = 0.9;
+export const TAKEOVER_RATE = 0.2;
+export const SCORE_CUTOFF = 0.01;
+
+/** Chime's active speaker ranking: who has been speaking recently, the most recent and steady first. */
+export class ActiveSpeakers {
+  private readonly scores = new Map<string, number>();
+  private last: string[] = [];
+
+  /**
+   * One sample: each identity's loudest level now (0..1). Identities missing from it count as silent. Returns the new
+   * order when it changed, else null.
+   */
+  update(levels: ReadonlyMap<string, number>): string[] | null {
+    for (const id of levels.keys()) if (!this.scores.has(id)) this.scores.set(id, 0);
+    // Chime updates one attendee at a time as volumes arrive; here everyone is sampled together, so people speaking
+    // at the same moment do not take over from each other -- only the silent ones lose ground.
+    const speakingNow = new Set([...levels].filter(([, level]) => level > SPEAKING_LEVEL).map(([id]) => id));
+    for (const id of this.scores.keys()) {
+      const speaking = speakingNow.has(id) ? 1 : 0;
+      let score = (this.scores.get(id) ?? 0) * SPEAKER_WEIGHT + speaking * (1 - SPEAKER_WEIGHT);
+      if (!speaking && speakingNow.size > 0) score = Math.max(score - TAKEOVER_RATE, 0);
+      this.scores.set(id, score);
+    }
+    const order = [...this.scores.entries()]
+      .filter(([, score]) => score >= SCORE_CUTOFF)
+      .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
+      .map(([id]) => id);
+    if (order.length === this.last.length && order.every((id, i) => id === this.last[i])) return null;
+    this.last = order;
+    return order;
+  }
+
+  /** Someone left: forget them. Returns the new order when it changed. */
+  remove(identity: string): string[] | null {
+    if (!this.scores.delete(identity)) return null;
+    const order = this.last.filter((id) => id !== identity);
+    if (order.length === this.last.length) return null;
+    this.last = order;
+    return order;
+  }
+
+  reset(): void {
+    this.scores.clear();
+    this.last = [];
+  }
+}