@voltkit-rtc/client 0.8.0 → 0.9.0

package/src/noise-suppression.ts

@@ -0,0 +1,135 @@
+/**
+ * AI noise suppression for the microphone (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-4): what Agora's AI
+ * Denoiser, LiveKit's Krisp filter and ZEGOCLOUD's AI noise suppression do, with RNNoise -- the open neural-network
+ * denoiser Jitsi uses -- through `@sapphi-red/web-noise-suppressor` (MIT): a Web Audio worklet in the browser's
+ * audio thread. Fans, traffic and room rumble fall away (measured in Chromium: rumble-like noise down by more than
+ * 60 dB, steady white hiss by 7-12 dB); voices stay.
+ *
+ * Imported on its own (`@voltkit-rtc/client/noise-suppression`), so an app that does not use it never ships it:
+ *
+ *   import { createNoiseSuppressor } from '@voltkit-rtc/client/noise-suppression';
+ *   await client.setAudioProcessor(createNoiseSuppressor());
+ *
+ * The worklet (64 kB) and the model (150 kB) are fetched when it starts: from jsDelivr, at the exact version this
+ * SDK was tested with, unless you serve them yourself (`workletUrl`, `wasmUrl`, `simdWasmUrl`) -- copy them from
+ * `node_modules/@sapphi-red/web-noise-suppressor/dist` to your own site for a strict Content-Security-Policy.
+ */
+import type { AudioTrackProcessor } from './processors.js';
+
+/** The package version the default URLs point at; equal to this SDK's pinned dependency (a test checks it). */
+export const NOISE_SUPPRESSOR_VERSION = '0.4.1';
+const CDN = `https://cdn.jsdelivr.net/npm/@sapphi-red/web-noise-suppressor@${NOISE_SUPPRESSOR_VERSION}/dist`;
+
+export interface NoiseSuppressorOptions {
+  /** Where `rnnoise/workletProcessor.js` is served. Default: jsDelivr, pinned. */
+  workletUrl?: string;
+  /** Where `rnnoise.wasm` is served. Default: jsDelivr, pinned. */
+  wasmUrl?: string;
+  /** Where `rnnoise_simd.wasm` is served (used where the browser supports SIMD). Default: jsDelivr, pinned. */
+  simdWasmUrl?: string;
+}
+
+/** What the processor needs from the browser; stood in for by tests. */
+export interface NoiseSuppressorDeps {
+  createContext(): AudioContext;
+  loadModel(urls: { url: string; simdUrl: string }): Promise<ArrayBuffer>;
+  createNode(context: AudioContext, wasmBinary: ArrayBuffer): Promise<DenoiseNode> | DenoiseNode;
+  createStream(tracks: MediaStreamTrack[]): MediaStream;
+}
+
+type DenoiseNode = AudioNode & { destroy?(): void };
+type Library = typeof import('@sapphi-red/web-noise-suppressor');
+let library: Promise<Library> | null = null;
+
+/**
+ * The library is loaded on first use, not with this module: it extends `AudioWorkletNode` as it loads, which exists
+ * only in browsers, so a static import would break a server render (Next.js) or Node.
+ */
+const load = (): Promise<Library> =>
+  (library ??= import('@sapphi-red/web-noise-suppressor').catch((err: unknown) => {
+    library = null;
+    throw err;
+  }));
+
+const browserDeps: NoiseSuppressorDeps = {
+  // RNNoise works on 48 kHz audio.
+  createContext: () => new AudioContext({ sampleRate: 48000 }),
+  loadModel: async (urls) => (await load()).loadRnnoise(urls),
+  createNode: async (context, wasmBinary) => new (await load()).RnnoiseWorkletNode(context, { wasmBinary, maxChannels: 1 }),
+  createStream: (tracks) => new MediaStream(tracks),
+};
+
+/** An `AudioTrackProcessor` for `client.setAudioProcessor()` that removes background noise with RNNoise. */
+export function createNoiseSuppressor(options: NoiseSuppressorOptions = {}, deps: NoiseSuppressorDeps = browserDeps): AudioTrackProcessor {
+  let context: AudioContext | null = null;
+  let ready: Promise<ArrayBuffer> | null = null;
+  let graph: { source: AudioNode; node: DenoiseNode; output: MediaStreamTrack } | null = null;
+  let resumeOnGesture: (() => void) | null = null;
+
+  const release = () => {
+    if (!graph) return;
+    try {
+      graph.source.disconnect();
+      graph.node.disconnect();
+      graph.node.destroy?.();
+    } catch {
+      // Already gone.
+    }
+    graph.output.stop();
+    graph = null;
+  };
+
+  /** A context made without a tap starts suspended: no sound would go out until the page is touched. */
+  const keepRunning = (ctx: AudioContext) => {
+    void ctx.resume?.().catch(() => undefined);
+    if (ctx.state === 'running' || typeof document === 'undefined' || resumeOnGesture) return;
+    resumeOnGesture = () => {
+      void ctx.resume().catch(() => undefined);
+      if (ctx.state === 'running' && resumeOnGesture) {
+        for (const type of ['pointerdown', 'keydown'] as const) document.removeEventListener(type, resumeOnGesture, { capture: true });
+        resumeOnGesture = null;
+      }
+    };
+    for (const type of ['pointerdown', 'keydown'] as const) document.addEventListener(type, resumeOnGesture, { capture: true });
+  };
+
+  return {
+    name: 'noise-suppression',
+    async process(track: MediaStreamTrack): Promise<MediaStreamTrack> {
+      const ctx = (context ??= deps.createContext());
+      ready ??= Promise.all([
+        deps.loadModel({ url: options.wasmUrl ?? `${CDN}/rnnoise.wasm`, simdUrl: options.simdWasmUrl ?? `${CDN}/rnnoise_simd.wasm` }),
+        ctx.audioWorklet.addModule(options.workletUrl ?? `${CDN}/rnnoise/workletProcessor.js`),
+      ]).then(([model]) => model);
+      let model: ArrayBuffer;
+      try {
+        model = await ready;
+      } catch (err) {
+        ready = null; // try again next time (a network blip)
+        throw err;
+      }
+      const node = await deps.createNode(ctx, model);
+      release();
+      const source = ctx.createMediaStreamSource(deps.createStream([track]));
+      const destination = ctx.createMediaStreamDestination();
+      source.connect(node);
+      node.connect(destination);
+      const output = destination.stream.getAudioTracks()[0];
+      if (!output) throw new Error('the audio graph gave no track');
+      graph = { source, node, output };
+      keepRunning(ctx);
+      return output;
+    },
+    async destroy(): Promise<void> {
+      release();
+      if (resumeOnGesture && typeof document !== 'undefined') {
+        for (const type of ['pointerdown', 'keydown'] as const) document.removeEventListener(type, resumeOnGesture, { capture: true });
+      }
+      resumeOnGesture = null;
+      const ctx = context;
+      context = null;
+      ready = null;
+      await ctx?.close().catch(() => undefined);
+    },
+  };
+}

package/src/processors.ts

@@ -0,0 +1,35 @@
+/**
+ * Track processors (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-4, P3-5): the published microphone or
+ * camera sent through something that changes it first -- noise suppression, a blurred or replaced background, beauty.
+ * LiveKit (`setProcessor`: Krisp, background blur), Agora and TRTC (extensions and plugins) do it the same way: the
+ * device's track goes in, a processed track goes out and is published in its place; the device's own track stays the
+ * one watched for unplugging, given new constraints and released at the end.
+ */
+
+/** Something the microphone passes through before it is sent. */
+export interface AudioTrackProcessor {
+  /** A short name, for diagnostics. */
+  readonly name: string;
+  /**
+   * Process `track` and return the track to send. Called again whenever the microphone's track changes (another
+   * device, a reconnect): the processor then lets go of what it made for the previous one.
+   */
+  process(track: MediaStreamTrack): Promise<MediaStreamTrack>;
+  /** Let go of everything (the processor was removed, or the session ended). */
+  destroy(): Promise<void> | void;
+}
+
+/** Something the camera passes through before it is sent (and shown in `client.local`). */
+export interface VideoTrackProcessor {
+  /** A short name, for diagnostics. */
+  readonly name: string;
+  /**
+   * Process `track` and return the track to send, at the same size and frame rate. Called again when the camera's
+   * track changes (another camera, `replaceVideoTrack`): the processor then lets go of what it made for the previous
+   * one. A picture the SDK no longer sends is stopped by the SDK; stop drawing it once its `readyState` is 'ended',
+   * and skip the work while it is switched off (`enabled` false: the camera is paused).
+   */
+  process(track: MediaStreamTrack): Promise<MediaStreamTrack>;
+  /** Let go of everything (the processor was removed, or the session ended). */
+  destroy(): Promise<void> | void;
+}