@werxt/livekit-components-react 2.9.20

package/src/hooks/useTrackToggle.ts

@@ -0,0 +1,93 @@
+import type { ToggleSource } from '@livekit/components-core';
+import { setupMediaToggle, setupManualToggle, log } from '@livekit/components-core';
+import * as React from 'react';
+import { type Room } from 'livekit-client';
+import type { TrackToggleProps } from '../components';
+import { useMaybeRoomContext } from '../context';
+import { mergeProps } from '../mergeProps';
+import { useObservableState } from './internal';
+
+/** @public */
+export interface UseTrackToggleProps<T extends ToggleSource>
+  extends Omit<TrackToggleProps<T>, 'showIcon'> {
+  room?: Room;
+}
+
+/**
+ * The `useTrackToggle` hook is used to implement the `TrackToggle` component and returns state
+ * and functionality of the given track.
+ *
+ * @example
+ * ```tsx
+ * const { buttonProps, enabled } = useTrackToggle(trackRef);
+ * return <button {...buttonProps}>{enabled ? 'disable' : 'enable'}</button>;
+ * ```
+ * @public
+ */
+export function useTrackToggle<T extends ToggleSource>({
+  source,
+  onChange,
+  initialState,
+  captureOptions,
+  publishOptions,
+  onDeviceError,
+  room,
+  ...rest
+}: UseTrackToggleProps<T>) {
+  const roomFromContext = useMaybeRoomContext();
+  const roomFallback = React.useMemo(() => room ?? roomFromContext, [room, roomFromContext]);
+  const track = roomFallback?.localParticipant?.getTrackPublication(source);
+  /** `true` if a user interaction such as a click on the TrackToggle button has occurred. */
+  const userInteractionRef = React.useRef(false);
+
+  const { toggle, className, pendingObserver, enabledObserver } = React.useMemo(
+    () =>
+      roomFallback
+        ? setupMediaToggle<T>(source, roomFallback, captureOptions, publishOptions, onDeviceError)
+        : setupManualToggle(),
+    [roomFallback, source, JSON.stringify(captureOptions), publishOptions],
+  );
+
+  const pending = useObservableState(pendingObserver, false);
+  const enabled = useObservableState(enabledObserver, initialState ?? !!track?.isEnabled);
+
+  React.useEffect(() => {
+    onChange?.(enabled, userInteractionRef.current);
+    userInteractionRef.current = false;
+  }, [enabled, onChange]);
+
+  React.useEffect(() => {
+    if (initialState !== undefined) {
+      log.debug('forcing initial toggle state', source, initialState);
+      toggle(initialState);
+    }
+    // only execute once at the beginning
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const newProps = React.useMemo(() => mergeProps(rest, { className }), [rest, className]);
+
+  const clickHandler: React.MouseEventHandler<HTMLButtonElement> = React.useCallback(
+    (evt) => {
+      userInteractionRef.current = true;
+      toggle().catch(() => (userInteractionRef.current = false));
+      rest.onClick?.(evt);
+    },
+    [rest, toggle],
+  );
+
+  return {
+    toggle,
+    enabled,
+    pending,
+    track,
+    buttonProps: {
+      ...newProps,
+      'aria-pressed': enabled,
+      'data-lk-source': source,
+      'data-lk-enabled': enabled,
+      disabled: pending,
+      onClick: clickHandler,
+    } as React.ButtonHTMLAttributes<HTMLButtonElement>,
+  };
+}

package/src/hooks/useTrackTranscription.ts

@@ -0,0 +1,75 @@
+import {
+  type ReceivedTranscriptionSegment,
+  addMediaTimestampToTranscription as addTimestampsToTranscription,
+  dedupeSegments,
+  // getActiveTranscriptionSegments,
+  getTrackReferenceId,
+  trackTranscriptionObserver,
+  type TrackReferenceOrPlaceholder,
+  // didActiveSegmentsChange,
+} from '@livekit/components-core';
+import type { TranscriptionSegment } from 'livekit-client';
+import * as React from 'react';
+import { useTrackSyncTime } from './useTrackSyncTime';
+
+/**
+ * @alpha
+ * @deprecated Use useTranscription instead
+ */
+export interface TrackTranscriptionOptions {
+  /**
+   * how many transcription segments should be buffered in state
+   * @defaultValue 100
+   */
+  bufferSize?: number;
+  /**
+   * optional callback for retrieving newly incoming transcriptions only
+   */
+  onTranscription?: (newSegments: TranscriptionSegment[]) => void;
+  /** amount of time (in ms) that the segment is considered `active` past its original segment duration, defaults to 2_000 */
+  // maxAge?: number;
+}
+
+const TRACK_TRANSCRIPTION_DEFAULTS = {
+  bufferSize: 100,
+  // maxAge: 2_000,
+} as const satisfies TrackTranscriptionOptions;
+
+/**
+ * @returns An object consisting of `segments` with maximum length of opts.bufferSize
+ * @alpha
+ * @deprecated Use useTranscription instead
+ */
+export function useTrackTranscription(
+  trackRef: TrackReferenceOrPlaceholder | undefined,
+  options?: TrackTranscriptionOptions,
+) {
+  const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options };
+  const [segments, setSegments] = React.useState<Array<ReceivedTranscriptionSegment>>([]);
+
+  const syncTimestamps = useTrackSyncTime(trackRef);
+  const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => {
+    opts.onTranscription?.(newSegments);
+    setSegments((prevSegments) =>
+      dedupeSegments(
+        prevSegments,
+        // when first receiving a segment, add the current media timestamp to it
+        newSegments.map((s) => addTimestampsToTranscription(s, syncTimestamps)),
+        opts.bufferSize,
+      ),
+    );
+  };
+  React.useEffect(() => {
+    if (!trackRef?.publication) {
+      return;
+    }
+    const subscription = trackTranscriptionObserver(trackRef.publication).subscribe((evt) => {
+      handleSegmentMessage(...evt);
+    });
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]);
+
+  return { segments };
+}

package/src/hooks/useTrackVolume.ts

@@ -0,0 +1,283 @@
+import * as React from 'react';
+import type { LocalAudioTrack, RemoteAudioTrack, AudioAnalyserOptions } from 'livekit-client';
+import { Track, createAudioAnalyser } from 'livekit-client';
+import {
+  type TrackReference,
+  isTrackReference,
+  type TrackReferenceOrPlaceholder,
+} from '@livekit/components-core';
+
+/**
+ * @alpha
+ * Hook for tracking the volume of an audio track using the Web Audio API.
+ */
+export function useTrackVolume(
+  trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReference,
+  options: AudioAnalyserOptions = { fftSize: 32, smoothingTimeConstant: 0 },
+) {
+  const track = isTrackReference(trackOrTrackReference)
+    ? <LocalAudioTrack | RemoteAudioTrack | undefined>trackOrTrackReference.publication.track
+    : trackOrTrackReference;
+  const [volume, setVolume] = React.useState(0);
+  React.useEffect(() => {
+    if (!track || !track.mediaStream) {
+      return;
+    }
+
+    const { cleanup, analyser } = createAudioAnalyser(track, options);
+
+    const bufferLength = analyser.frequencyBinCount;
+    const dataArray = new Uint8Array(bufferLength);
+
+    const updateVolume = () => {
+      analyser.getByteFrequencyData(dataArray);
+      let sum = 0;
+      for (let i = 0; i < dataArray.length; i++) {
+        const a = dataArray[i];
+        sum += a * a;
+      }
+      setVolume(Math.sqrt(sum / dataArray.length) / 255);
+    };
+
+    const interval = setInterval(updateVolume, 1000 / 30);
+
+    return () => {
+      cleanup();
+      clearInterval(interval);
+    };
+  }, [track, track?.mediaStream, JSON.stringify(options)]);
+
+  return volume;
+}
+
+const normalizeFrequencies = (frequencies: Float32Array) => {
+  const normalizeDb = (value: number) => {
+    const minDb = -100;
+    const maxDb = -10;
+    let db = 1 - (Math.max(minDb, Math.min(maxDb, value)) * -1) / 100;
+    db = Math.sqrt(db);
+
+    return db;
+  };
+
+  // Normalize all frequency values
+  return frequencies.map((value) => {
+    if (value === -Infinity) {
+      return 0;
+    }
+    return normalizeDb(value);
+  });
+};
+
+/**
+ * Interface for configuring options for the useMultibandTrackVolume hook.
+ * @alpha
+ */
+export interface MultiBandTrackVolumeOptions {
+  bands?: number;
+  /**
+   * cut off of frequency bins on the lower end
+   * Note: this is not a frequency measure, but in relation to analyserOptions.fftSize,
+   */
+  loPass?: number;
+  /**
+   * cut off of frequency bins on the higher end
+   * Note: this is not a frequency measure, but in relation to analyserOptions.fftSize,
+   */
+  hiPass?: number;
+  /**
+   * update should run every x ms
+   */
+  updateInterval?: number;
+  analyserOptions?: AnalyserOptions;
+}
+
+const multibandDefaults = {
+  bands: 5,
+  loPass: 100,
+  hiPass: 600,
+  updateInterval: 32,
+  analyserOptions: { fftSize: 2048 },
+} as const satisfies MultiBandTrackVolumeOptions;
+
+/**
+ * Hook for tracking the volume of an audio track across multiple frequency bands using the Web Audio API.
+ * @alpha
+ */
+export function useMultibandTrackVolume(
+  trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder,
+  options: MultiBandTrackVolumeOptions = {},
+) {
+  const track =
+    trackOrTrackReference instanceof Track
+      ? trackOrTrackReference
+      : <LocalAudioTrack | RemoteAudioTrack | undefined>trackOrTrackReference?.publication?.track;
+  const opts = { ...multibandDefaults, ...options };
+  const [frequencyBands, setFrequencyBands] = React.useState<Array<number>>(
+    new Array(opts.bands).fill(0),
+  );
+
+  React.useEffect(() => {
+    if (!track || !track?.mediaStream) {
+      setFrequencyBands((val) => val.slice().fill(0));
+      return;
+    }
+    const { analyser, cleanup } = createAudioAnalyser(track, opts.analyserOptions);
+
+    const bufferLength = analyser.frequencyBinCount;
+    const dataArray = new Float32Array(bufferLength);
+
+    const updateVolume = () => {
+      analyser.getFloatFrequencyData(dataArray);
+      let frequencies: Float32Array = new Float32Array(dataArray.length);
+      for (let i = 0; i < dataArray.length; i++) {
+        frequencies[i] = dataArray[i];
+      }
+      frequencies = frequencies.slice(opts.loPass, opts.hiPass);
+
+      const normalizedFrequencies = normalizeFrequencies(frequencies); // is this needed ?
+      const totalBins = normalizedFrequencies.length;
+      const chunks: Array<number> = [];
+      for (let i = 0; i < opts.bands; i++) {
+        // Use proportional distribution to evenly divide bins across bands
+        const startIndex = Math.floor((i * totalBins) / opts.bands);
+        const endIndex = Math.floor(((i + 1) * totalBins) / opts.bands);
+        const chunk = normalizedFrequencies.slice(startIndex, endIndex);
+        const chunkLength = chunk.length;
+        if (chunkLength === 0) {
+          chunks.push(0);
+        } else {
+          const summedVolumes = chunk.reduce((acc, val) => (acc += val), 0);
+          chunks.push(summedVolumes / chunkLength);
+        }
+      }
+
+      setFrequencyBands(chunks);
+    };
+
+    const interval = setInterval(updateVolume, opts.updateInterval);
+
+    return () => {
+      cleanup();
+      clearInterval(interval);
+    };
+  }, [track, track?.mediaStream, JSON.stringify(options)]);
+
+  return frequencyBands;
+}
+
+/**
+ * @alpha
+ */
+export interface AudioWaveformOptions {
+  barCount?: number;
+  volMultiplier?: number;
+  updateInterval?: number;
+}
+
+const waveformDefaults = {
+  barCount: 120,
+  volMultiplier: 5,
+  updateInterval: 20,
+} as const satisfies AudioWaveformOptions;
+
+/**
+ * @alpha
+ */
+export function useAudioWaveform(
+  trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder,
+  options: AudioWaveformOptions = {},
+) {
+  const track =
+    trackOrTrackReference instanceof Track
+      ? trackOrTrackReference
+      : <LocalAudioTrack | RemoteAudioTrack | undefined>trackOrTrackReference?.publication?.track;
+  const opts = { ...waveformDefaults, ...options };
+
+  const aggregateWave = React.useRef(new Float32Array());
+  const timeRef = React.useRef(performance.now());
+  const updates = React.useRef(0);
+  const [bars, setBars] = React.useState<number[]>([]);
+
+  const onUpdate = React.useCallback((wave: Float32Array) => {
+    setBars(
+      Array.from(
+        filterData(wave, opts.barCount).map((v) => Math.sqrt(v) * opts.volMultiplier),
+        // wave.slice(0, opts.barCount).map((v) => sigmoid(v * opts.volMultiplier, 0.08, 0.2)),
+      ),
+    );
+  }, []);
+
+  React.useEffect(() => {
+    if (!track || !track?.mediaStream) {
+      return;
+    }
+    const { analyser, cleanup } = createAudioAnalyser(track, {
+      fftSize: getFFTSizeValue(opts.barCount),
+    });
+
+    const bufferLength = getFFTSizeValue(opts.barCount);
+    const dataArray = new Float32Array(bufferLength);
+
+    const update = () => {
+      updateWaveform = requestAnimationFrame(update);
+      analyser.getFloatTimeDomainData(dataArray);
+      aggregateWave.current.map((v, i) => v + dataArray[i]);
+      updates.current += 1;
+
+      if (performance.now() - timeRef.current >= opts.updateInterval) {
+        const newData = dataArray.map((v) => v / updates.current);
+        onUpdate(newData);
+        timeRef.current = performance.now();
+        updates.current = 0;
+      }
+    };
+
+    let updateWaveform = requestAnimationFrame(update);
+
+    return () => {
+      cleanup();
+      cancelAnimationFrame(updateWaveform);
+    };
+  }, [track, track?.mediaStream, JSON.stringify(options), onUpdate]);
+
+  return {
+    bars,
+  };
+}
+
+function getFFTSizeValue(x: number) {
+  if (x < 32) return 32;
+  else return pow2ceil(x);
+}
+
+// function sigmoid(x: number, k = 2, s = 0) {
+//   return 1 / (1 + Math.exp(-(x - s) / k));
+// }
+
+function pow2ceil(v: number) {
+  let p = 2;
+  while ((v >>= 1)) {
+    p <<= 1;
+  }
+  return p;
+}
+
+function filterData(audioData: Float32Array, numSamples: number) {
+  const blockSize = Math.floor(audioData.length / numSamples); // the number of samples in each subdivision
+  const filteredData = new Float32Array(numSamples);
+  for (let i = 0; i < numSamples; i++) {
+    const blockStart = blockSize * i; // the location of the first sample in the block
+    let sum = 0;
+    for (let j = 0; j < blockSize; j++) {
+      sum = sum + Math.abs(audioData[blockStart + j]); // find the sum of all the samples in the block
+    }
+    filteredData[i] = sum / blockSize; // divide the sum by the block size to get the average
+  }
+  return filteredData;
+}
+
+// function normalizeData(audioData: Float32Array) {
+//   const multiplier = Math.pow(Math.max(...audioData), -1);
+//   return audioData.map((n) => n * multiplier);
+// }

package/src/hooks/useTracks.test.ts

@@ -0,0 +1,60 @@
+import { Participant, Track } from 'livekit-client';
+import { describe, test, expect } from 'vitest';
+import { requiredPlaceholders } from './useTracks';
+
+describe('Test requiredPlaceholders functions indicates that placeholders are required.', () => {
+  test('Participants without tracks should be included in the map.', () => {
+    const sources = [{ source: Track.Source.Camera, withPlaceholder: true }];
+    const participants = [new Participant('sid_A', 'identity_A')];
+    const results = requiredPlaceholders(sources, participants);
+    expect(results.get('identity_A')).toStrictEqual([Track.Source.Camera]);
+  });
+
+  test("Multiple participants should be included in the map if they don't have a subscribed tracks.", () => {
+    const sources = [
+      { source: Track.Source.Camera, withPlaceholder: true },
+      { source: Track.Source.ScreenShare, withPlaceholder: true },
+    ];
+    const participants = [
+      new Participant('sid_A', 'identity_A'),
+      new Participant('sid_B', 'identity_B'),
+    ];
+    const results = requiredPlaceholders(sources, participants);
+    expect(results.get('identity_A')).toStrictEqual([
+      Track.Source.Camera,
+      Track.Source.ScreenShare,
+    ]);
+    expect(results.get('identity_B')).toStrictEqual([
+      Track.Source.Camera,
+      Track.Source.ScreenShare,
+    ]);
+  });
+
+  test('The map should only include placeholders for if `withPlaceholder: true`', () => {
+    const sources = [
+      { source: Track.Source.Camera, withPlaceholder: true },
+      { source: Track.Source.ScreenShare, withPlaceholder: false },
+    ];
+    const participants = [
+      new Participant('sid_A', 'identity_A'),
+      new Participant('sid_B', 'identity_B'),
+    ];
+    const results = requiredPlaceholders(sources, participants);
+    expect(results.get('identity_A')).toStrictEqual([Track.Source.Camera]);
+    expect(results.get('identity_B')).toStrictEqual([Track.Source.Camera]);
+  });
+  test('If no placeholders are wanted the map should be empty', () => {
+    const sources = [
+      { source: Track.Source.Camera, withPlaceholder: false },
+      { source: Track.Source.ScreenShare, withPlaceholder: false },
+    ];
+    const participants = [
+      new Participant('sid_A', 'identity_A'),
+      new Participant('sid_B', 'identity_B'),
+    ];
+    const results = requiredPlaceholders(sources, participants);
+    expect(results.size).toBe(0);
+    expect(results.has('identity_A')).toBeFalsy();
+    expect(results.has('identity_B')).toBeFalsy();
+  });
+});

package/src/hooks/useTracks.ts

@@ -0,0 +1,154 @@
+import type {
+  SourcesArray,
+  TrackReference,
+  TrackReferenceOrPlaceholder,
+  TrackSourceWithOptions,
+  TrackReferencePlaceholder,
+} from '@livekit/components-core';
+import {
+  isSourcesWithOptions,
+  isSourceWitOptions,
+  log,
+  trackReferencesObservable,
+} from '@livekit/components-core';
+import type { Participant, Room, RoomEvent } from 'livekit-client';
+import { Track } from 'livekit-client';
+import * as React from 'react';
+import { useEnsureRoom } from '../context';
+
+/** @public */
+export type UseTracksOptions = {
+  updateOnlyOn?: RoomEvent[];
+  onlySubscribed?: boolean;
+  room?: Room;
+};
+
+/** @public */
+export type UseTracksHookReturnType<T> = T extends Track.Source[]
+  ? TrackReference[]
+  : T extends TrackSourceWithOptions[]
+    ? TrackReferenceOrPlaceholder[]
+    : never;
+
+/**
+ * The `useTracks` hook returns an array of `TrackReference` or `TrackReferenceOrPlaceholder` depending on the provided `sources` property.
+ * If only subscribed tracks are desired, set the `onlySubscribed` property to `true`.
+ * @example
+ * ```ts
+ * // Return all camera track publications.
+ * const trackReferences: TrackReference[] = useTracks([Track.Source.Camera])
+ * ```
+ * @example
+ * ```ts
+ * // Return all subscribed camera tracks as well as placeholders for
+ * // participants without a camera subscription.
+ * const trackReferencesWithPlaceholders: TrackReferenceOrPlaceholder[] = useTracks([{source: Track.Source.Camera, withPlaceholder: true}])
+ * ```
+ * @public
+ */
+export function useTracks<T extends SourcesArray = Track.Source[]>(
+  sources: T = [
+    Track.Source.Camera,
+    Track.Source.Microphone,
+    Track.Source.ScreenShare,
+    Track.Source.ScreenShareAudio,
+    Track.Source.Unknown,
+  ] as T,
+  options: UseTracksOptions = {},
+): UseTracksHookReturnType<T> {
+  const room = useEnsureRoom(options.room);
+  const [trackReferences, setTrackReferences] = React.useState<TrackReference[]>([]);
+  const [participants, setParticipants] = React.useState<Participant[]>([]);
+
+  const sources_ = React.useMemo(() => {
+    return sources.map((s) => (isSourceWitOptions(s) ? s.source : s));
+  }, [JSON.stringify(sources)]);
+
+  React.useEffect(() => {
+    const subscription = trackReferencesObservable(room, sources_, {
+      additionalRoomEvents: options.updateOnlyOn,
+      onlySubscribed: options.onlySubscribed,
+    }).subscribe(({ trackReferences, participants }) => {
+      log.debug('setting track bundles', trackReferences, participants);
+      setTrackReferences(trackReferences);
+      setParticipants(participants);
+    });
+    return () => subscription.unsubscribe();
+  }, [
+    room,
+    JSON.stringify(options.onlySubscribed),
+    JSON.stringify(options.updateOnlyOn),
+    JSON.stringify(sources),
+  ]);
+
+  const maybeTrackReferences = React.useMemo(() => {
+    if (isSourcesWithOptions(sources)) {
+      const requirePlaceholder = requiredPlaceholders(sources, participants);
+      const trackReferencesWithPlaceholders: TrackReferenceOrPlaceholder[] =
+        Array.from(trackReferences);
+      participants.forEach((participant) => {
+        if (requirePlaceholder.has(participant.identity)) {
+          const sourcesToAddPlaceholder = requirePlaceholder.get(participant.identity) ?? [];
+          sourcesToAddPlaceholder.forEach((placeholderSource) => {
+            if (
+              trackReferences.find(
+                ({ participant: p, publication }) =>
+                  participant.identity === p.identity && publication.source === placeholderSource,
+              )
+            ) {
+              return;
+            }
+            log.debug(
+              `Add ${placeholderSource} placeholder for participant ${participant.identity}.`,
+            );
+            const placeholder: TrackReferencePlaceholder = {
+              participant,
+              source: placeholderSource,
+            };
+            trackReferencesWithPlaceholders.push(placeholder);
+          });
+        }
+      });
+      return trackReferencesWithPlaceholders;
+    } else {
+      return trackReferences;
+    }
+  }, [trackReferences, participants, sources]);
+
+  return maybeTrackReferences as UseTracksHookReturnType<T>;
+}
+
+function difference<T>(setA: Set<T>, setB: Set<T>): Set<T> {
+  const _difference = new Set(setA);
+  for (const elem of setB) {
+    _difference.delete(elem);
+  }
+  return _difference;
+}
+
+export function requiredPlaceholders<T extends SourcesArray>(
+  sources: T,
+  participants: Participant[],
+): Map<Participant['identity'], Track.Source[]> {
+  const placeholderMap = new Map<Participant['identity'], Track.Source[]>();
+  if (isSourcesWithOptions(sources)) {
+    const sourcesThatNeedPlaceholder = sources
+      .filter((sourceWithOption) => sourceWithOption.withPlaceholder)
+      .map((sourceWithOption) => sourceWithOption.source);
+
+    participants.forEach((participant) => {
+      const sourcesOfSubscribedTracks = participant
+        .getTrackPublications()
+        .map((pub) => pub.track?.source)
+        .filter((trackSource): trackSource is Track.Source => trackSource !== undefined);
+      const placeholderNeededForThisParticipant = Array.from(
+        difference(new Set(sourcesThatNeedPlaceholder), new Set(sourcesOfSubscribedTracks)),
+      );
+      // If the participant needs placeholder add it to the placeholder map.
+      if (placeholderNeededForThisParticipant.length > 0) {
+        placeholderMap.set(participant.identity, placeholderNeededForThisParticipant);
+      }
+    });
+  }
+  return placeholderMap;
+}