@voltkit-rtc/client 0.7.2 → 0.8.0

package/src/diagnostics.ts

@@ -0,0 +1,94 @@
+/**
+ * Diagnostics for support (docs/implementation-progress/RTC_PERF_PROGRESS.md S4).
+ *
+ * Agora uploads its SDK log on request (`enableLogUpload`), Twilio sends call telemetry to Insights, Stream Video
+ * traces session events to its servers. Shipping telemetry needs a backend; what helps today is a snapshot an app can
+ * attach to a support ticket: the SDK's recent events (a ring buffer) and its state now.
+ */
+
+/** This package's version; kept equal to package.json by a test. */
+export const SDK_VERSION = '0.8.0';
+
+/** How many recent events the log keeps. */
+export const DIAGNOSTICS_LOG_SIZE = 200;
+
+export interface DiagnosticsEntry {
+  /** Unix ms. */
+  at: number;
+  event: string;
+  /** A short summary of what the event carried (no media objects). */
+  detail?: string;
+}
+
+/** The events worth keeping for support, and how to summarise each. */
+const SUMMARY: Record<string, (args: unknown[]) => string | undefined> = {
+  connectionStateChanged: ([e]) => { const s = e as { previous?: string; state?: string }; return `${s.previous} → ${s.state}`; },
+  disconnected: ([reason]) => String(reason),
+  closed: ([reason]) => String(reason),
+  mustReconnect: ([e]) => (e as { reason?: string }).reason,
+  mediaReconnecting: ([e]) => { const m = e as { transportId?: string; reconnecting?: boolean }; return `${m.transportId} ${m.reconnecting ? 'recovering' : 'recovered'}`; },
+  error: ([e]) => {
+    const err = e as { code?: string; message?: string };
+    const message = err.message ?? '';
+    // RpcError's message already starts with its code.
+    return (err.code && !message.startsWith(`${err.code}:`) ? `${err.code}: ${message}` : message).slice(0, 300);
+  },
+  streamFallback: ([e]) => { const f = e as { producerId?: string; fallback?: boolean; reason?: string }; return `${f.producerId} ${f.fallback ? 'held' : 'back'} (${f.reason})`; },
+  mediaWarning: ([e]) => { const w = e as { code?: string; active?: boolean }; return `${w.code} ${w.active ? 'started' : 'cleared'}`; },
+  cpuConstrained: ([e]) => String((e as { constrained?: boolean }).constrained),
+  videoLimited: ([e]) => (e as { reason?: string }).reason,
+  localTrackEnded: ([e]) => { const t = e as { kind?: string; recovered?: boolean }; return `${t.kind} ${t.recovered ? 'recovered' : 'off'}`; },
+  devicesChanged: ([e]) => { const d = e as { added?: unknown[]; removed?: unknown[] }; return `+${d.added?.length ?? 0} -${d.removed?.length ?? 0}`; },
+  kicked: ([e]) => (e as { reason?: string }).reason,
+  roomClosed: ([e]) => (e as { reason?: string }).reason,
+  usageLimit: ([e]) => (e as { reason?: string }).reason,
+  autoplayFailed: () => undefined,
+  trackEnded: ([e]) => (e as { producerId?: string }).producerId,
+  track: ([e]) => { const t = e as { identity?: string; source?: string }; return `${t.identity} ${t.source}`; },
+};
+
+/** A ring buffer of the SDK's recent events. */
+export class DiagnosticsLog {
+  private readonly entries: DiagnosticsEntry[] = [];
+
+  constructor(private readonly now: () => number = Date.now) {}
+
+  /** Keep an emitted event, when it is one worth keeping. */
+  record(event: string, args: unknown[]): void {
+    const summarize = SUMMARY[event];
+    if (!summarize) return;
+    let detail: string | undefined;
+    try {
+      detail = summarize(args);
+    } catch {
+      detail = undefined;
+    }
+    this.entries.push(detail === undefined ? { at: this.now(), event } : { at: this.now(), event, detail });
+    if (this.entries.length > DIAGNOSTICS_LOG_SIZE) this.entries.shift();
+  }
+
+  list(): DiagnosticsEntry[] {
+    return this.entries.map((e) => ({ ...e }));
+  }
+}
+
+export interface DiagnosticsTransport {
+  state: string;
+  path: { protocol: 'udp' | 'tcp' | null; relayed: boolean } | null;
+}
+
+export interface Diagnostics {
+  sdk: string;
+  at: number;
+  userAgent: string;
+  connectionState: string;
+  room: string | null;
+  identity: string | null;
+  role: string | null;
+  transports: { send: DiagnosticsTransport | null; recv: DiagnosticsTransport | null };
+  network: { rttMs: number | null; availableKbps: number | null } | null;
+  published: { source: string; kind: string; paused: boolean }[];
+  received: { identity: string; source: string; kind: string; paused: boolean }[];
+  /** Recent events, oldest first. */
+  log: DiagnosticsEntry[];
+}

package/src/fallback.ts

@@ -0,0 +1,169 @@
+/**
+ * Audio-only fallback (docs/implementation-progress/RTC_PERF_PROGRESS.md H1).
+ *
+ * When this viewer's connection is so poor that even the sound breaks up, remote camera videos are paused at the
+ * server so what bandwidth there is goes to the audio, and they come back once the connection has recovered.
+ *
+ * - Agora's `setStreamFallbackOption(uid, AUDIO_ONLY)` drops to the low stream and then to audio only, and reports
+ *   `stream-fallback` (fallback / recover).
+ * - Amazon Chime watches the audio it receives (`UnusableAudioWarningConnectionHealthPolicy`): about half of the
+ *   expected packets missing over 15 one-second samples suggests stopping video, with a 60 s cool-down.
+ *
+ * This follows Chime's signal -- the audio actually received -- measured as RTP loss (sequence gaps) rather than
+ * packet counts, because Opus DTX sends almost nothing during silence and a count would read silence as loss.
+ * Extreme only: {@link FALLBACK_LOSS} of the audio lost over {@link FALLBACK_SAMPLES} samples, with at least
+ * {@link MIN_PACKETS} packets to judge by. Back after a hold ({@link FIRST_HOLD_MS}, doubled for each relapse within
+ * {@link RELAPSE_MS}, up to {@link MAX_HOLD_MS}) once the loss is under {@link RECOVER_LOSS} -- or nobody spoke
+ * enough to tell, in which case video comes back and a still-bad connection falls back again, holding longer.
+ *
+ * The SFU cutting a video for bandwidth by itself is reported separately, by the server (`reason: 'bandwidth'`).
+ * Pure: the decisions take counters and a clock.
+ */
+import type { StatsEntry } from './stats.js';
+
+/** How often the received audio is sampled while the fallback could be needed. */
+export const PROTECT_SAMPLE_MS = 3000;
+/** This share of the audio lost (RTP sequence gaps) over the window: fall back. */
+export const FALLBACK_LOSS = 0.25;
+/** Below this share lost over the recovery window: video may come back. */
+export const RECOVER_LOSS = 0.05;
+/** Samples judged for falling back (15 s at 3 s) -- at least 3 of them. */
+export const FALLBACK_SAMPLES = 5;
+/** Samples judged for coming back (12 s). */
+export const RECOVER_SAMPLES = 4;
+/** Fewer audio packets than this over the fallback window are too few to judge. */
+export const MIN_PACKETS = 150;
+/** Fewer than this over the recovery window: nobody spoke enough to tell. */
+export const MIN_RECOVER_PACKETS = 30;
+/** The first fallback lasts at least this long. */
+export const FIRST_HOLD_MS = 20_000;
+/** A relapse doubles the hold, up to this. */
+export const MAX_HOLD_MS = 160_000;
+/** Falling back again within this long of coming back is a relapse. */
+export const RELAPSE_MS = 120_000;
+
+export interface AudioCounters {
+  received: number;
+  lost: number;
+}
+
+const num = (value: unknown): number => (typeof value === 'number' && Number.isFinite(value) ? value : 0);
+
+/** The received-audio counters of a receive transport's report, summed over its audio streams. */
+export function readInboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let received = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null || e.type !== 'inbound-rtp') continue;
+    if (e.kind !== 'audio' && e.mediaType !== 'audio') continue;
+    received += num(e.packetsReceived);
+    lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received, lost };
+}
+
+/**
+ * The sent-audio counters of a send transport's report: packets sent, and how many of them the server reported
+ * lost (its RTCP receiver reports). For the publisher side (Agora's setLocalPublishFallbackOption): `received` is what
+ * got through.
+ */
+export function readOutboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let sent = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null) continue;
+    const audio = e.kind === 'audio' || e.mediaType === 'audio';
+    if (!audio) continue;
+    if (e.type === 'outbound-rtp') sent += num(e.packetsSent);
+    else if (e.type === 'remote-inbound-rtp') lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received: Math.max(0, sent - lost), lost };
+}
+
+/** Decides when to fall back to audio only and when video may come back. */
+export class AudioProtection {
+  private samples: AudioCounters[] = [];
+  private previous: AudioCounters | null = null;
+  private activeSince: number | null = null;
+  private hold = FIRST_HOLD_MS;
+  private recoveredAt: number | null = null;
+
+  constructor(
+    private readonly onChange: (active: boolean) => void,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  /** Whether remote camera videos are held back now. */
+  get active(): boolean {
+    return this.activeSince !== null;
+  }
+
+  /** How long the current (or next first) fallback holds, in ms. */
+  get holdMs(): number {
+    return this.hold;
+  }
+
+  /**
+   * One reading of the cumulative counters. They only go up on one transport; lower ones are a new transport, and
+   * judging starts over from them.
+   */
+  feed(counters: AudioCounters): void {
+    const before = this.previous;
+    this.previous = counters;
+    if (before === null || counters.received < before.received || counters.lost < before.lost) {
+      this.samples = [];
+      return;
+    }
+    this.samples.push({ received: counters.received - before.received, lost: counters.lost - before.lost });
+    if (this.samples.length > Math.max(FALLBACK_SAMPLES, RECOVER_SAMPLES)) this.samples.shift();
+
+    if (this.activeSince === null) {
+      const window = this.samples.slice(-FALLBACK_SAMPLES);
+      if (window.length < 3) return;
+      const { total, lost } = sum(window);
+      if (total >= MIN_PACKETS && lost / total >= FALLBACK_LOSS) this.activate();
+      return;
+    }
+    if (this.now() - this.activeSince < this.hold || this.samples.length < RECOVER_SAMPLES) return;
+    const { total, lost } = sum(this.samples.slice(-RECOVER_SAMPLES));
+    if (total < MIN_RECOVER_PACKETS || lost / total < RECOVER_LOSS) this.recover();
+  }
+
+  /** Come back now (the app turned the fallback off). */
+  release(): void {
+    if (this.activeSince !== null) this.recover();
+  }
+
+  /** The transports went away (reconnect, close): forget everything, without telling. */
+  reset(): void {
+    this.activeSince = null;
+    this.samples = [];
+    this.previous = null;
+  }
+
+  private activate(): void {
+    const now = this.now();
+    const relapse = this.recoveredAt !== null && now - this.recoveredAt < RELAPSE_MS;
+    this.hold = relapse ? Math.min(this.hold * 2, MAX_HOLD_MS) : FIRST_HOLD_MS;
+    this.activeSince = now;
+    this.samples = [];
+    this.onChange(true);
+  }
+
+  private recover(): void {
+    this.activeSince = null;
+    this.recoveredAt = this.now();
+    this.samples = [];
+    this.onChange(false);
+  }
+}
+
+function sum(samples: readonly AudioCounters[]): { total: number; lost: number } {
+  let total = 0;
+  let lost = 0;
+  for (const s of samples) {
+    total += s.received + s.lost;
+    lost += s.lost;
+  }
+  return { total, lost };
+}

package/src/health.ts

@@ -0,0 +1,127 @@
+/**
+ * Media health warnings (docs/implementation-progress/RTC_PERF_PROGRESS.md K1).
+ *
+ * - Agora's `exception` event pairs each problem with its recovery: FRAMERATE_INPUT_TOO_LOW / _RECOVER (the camera
+ *   delivers too few frames), AUDIO_INPUT_LEVEL_TOO_LOW / _RECOVER, ...
+ * - LiveKit's `AudioSilenceDetected`: a microphone that delivers nothing but silence (a hardware mute switch, a
+ *   broken device) right after it starts.
+ * - LiveKit watches the track's `mute` event (the OS took the device: a phone call, another app) and reacts after 5 s.
+ *
+ * Each warning is reported once when it starts (`active: true`) and once when it clears (`active: false`).
+ * Pure: the decisions take samples.
+ */
+
+export type MediaWarningCode =
+  /** The microphone is on but delivers exact digital silence: a mute switch, a broken or wrongly chosen device. */
+  | 'mic-silent'
+  /** The camera delivers far fewer frames than asked: usually a dark room (the camera exposes longer), or a busy device. */
+  | 'camera-fps-low'
+  /** The system took the camera from this page for now (a phone call, another app, a locked screen). */
+  | 'camera-interrupted'
+  /** The system took the microphone (a phone call, another app). */
+  | 'mic-interrupted';
+
+export interface MediaWarning {
+  code: MediaWarningCode;
+  /** true when it starts, false when it clears. */
+  active: boolean;
+}
+
+/** How often the published camera and microphone are sampled while someone listens. */
+export const HEALTH_SAMPLE_MS = 2000;
+/** Samples of exact silence before mic-silent (6 s). */
+export const SILENT_SAMPLES = 3;
+/** Below this the level counts as exact digital silence (a real room's noise floor is well above it). */
+export const SILENCE_LEVEL = 0.0001;
+/** Fewer frames than this per second, when 24 or more were asked, is camera-fps-low. */
+export const LOW_FPS = 15;
+/** Samples below LOW_FPS before warning (6 s); samples at RECOVERED_FPS or more before clearing. */
+export const LOW_FPS_SAMPLES = 3;
+export const RECOVERED_FPS = 20;
+export const RECOVER_SAMPLES = 2;
+/** A system mute shorter than this is a glitch, not an interruption. */
+export const INTERRUPTED_AFTER_MS = 2000;
+
+/** One reading of the published devices. null: not published (or nothing measured). */
+export interface HealthSample {
+  /** The microphone's level, while it is on. */
+  micLevel: number | null;
+  /** The frames per second the camera delivers, while it is on, and what was asked. */
+  cameraFps: number | null;
+  askedFps: number;
+}
+
+/** Decides mic-silent and camera-fps-low from samples. */
+export class HealthDecisions {
+  private silent = 0;
+  private lowFps = 0;
+  private goodFps = 0;
+  private readonly activeNow = new Set<MediaWarningCode>();
+
+  constructor(private readonly report: (warning: MediaWarning) => void) {}
+
+  isActive(code: MediaWarningCode): boolean {
+    return this.activeNow.has(code);
+  }
+
+  feed(sample: HealthSample): void {
+    // mic-silent: exact silence held; any sound clears it. Off or unpublished: not a warning.
+    if (sample.micLevel === null) {
+      this.silent = 0;
+      this.set('mic-silent', false);
+    } else if (sample.micLevel <= SILENCE_LEVEL) {
+      this.silent += 1;
+      if (this.silent >= SILENT_SAMPLES) this.set('mic-silent', true);
+    } else {
+      this.silent = 0;
+      this.set('mic-silent', false);
+    }
+
+    // camera-fps-low: only when a smooth rate was asked; cleared once it holds again.
+    if (sample.cameraFps === null || sample.askedFps < 24) {
+      this.lowFps = 0;
+      this.goodFps = 0;
+      this.set('camera-fps-low', false);
+    } else if (sample.cameraFps < LOW_FPS) {
+      this.goodFps = 0;
+      this.lowFps += 1;
+      if (this.lowFps >= LOW_FPS_SAMPLES) this.set('camera-fps-low', true);
+    } else {
+      this.lowFps = 0;
+      if (sample.cameraFps >= RECOVERED_FPS) this.goodFps += 1;
+      if (this.goodFps >= RECOVER_SAMPLES) this.set('camera-fps-low', false);
+    }
+  }
+
+  /** A system interruption started or ended (decided by the caller's timer). */
+  interrupted(code: 'camera-interrupted' | 'mic-interrupted', active: boolean): void {
+    this.set(code, active);
+  }
+
+  /** Everything clears (unpublished, closed): each active warning is reported cleared. */
+  clearAll(): void {
+    for (const code of [...this.activeNow]) this.set(code, false);
+    this.silent = 0;
+    this.lowFps = 0;
+    this.goodFps = 0;
+  }
+
+  private set(code: MediaWarningCode, active: boolean): void {
+    if (this.activeNow.has(code) === active) return;
+    if (active) this.activeNow.add(code);
+    else this.activeNow.delete(code);
+    this.report({ code, active });
+  }
+}
+
+/** The microphone level and camera frame rate from a producer's statistics (media-source). */
+export function readMediaSource(report: Iterable<Record<string, unknown>>): { level: number | null; fps: number | null } {
+  let level: number | null = null;
+  let fps: number | null = null;
+  for (const s of report) {
+    if (s.type !== 'media-source') continue;
+    if (typeof s.audioLevel === 'number') level = s.audioLevel;
+    if (typeof s.framesPerSecond === 'number') fps = s.framesPerSecond;
+  }
+  return { level, fps };
+}