@livekit/react-native 2.5.1 → 2.6.0

package/src/components/BarVisualizer.tsx

@@ -0,0 +1,252 @@
+import {
+  type AgentState,
+  type TrackReferenceOrPlaceholder,
+  useMaybeTrackRefContext,
+} from '@livekit/components-react';
+import {
+  Animated,
+  StyleSheet,
+  View,
+  type ColorValue,
+  type DimensionValue,
+  type ViewStyle,
+} from 'react-native';
+import { useMultibandTrackVolume } from '../hooks';
+import React, { useEffect, useRef, useState } from 'react';
+export type BarVisualizerOptions = {
+  /** decimal values from 0 to 1 */
+  maxHeight?: number;
+  /** decimal values from 0 to 1 */
+  minHeight?: number;
+
+  barColor?: ColorValue;
+  barWidth?: DimensionValue;
+  barBorderRadius?: number;
+};
+
+const defaultBarOptions = {
+  maxHeight: 1,
+  minHeight: 0.2,
+  barColor: '#888888',
+  barWidth: 24,
+  barBorderRadius: 12,
+} as const satisfies BarVisualizerOptions;
+
+const sequencerIntervals = new Map<AgentState, number>([
+  ['connecting', 2000],
+  ['initializing', 2000],
+  ['listening', 500],
+  ['thinking', 150],
+]);
+
+const getSequencerInterval = (
+  state: AgentState | undefined,
+  barCount: number
+): number | undefined => {
+  if (state === undefined) {
+    return 1000;
+  }
+  let interval = sequencerIntervals.get(state);
+  if (interval) {
+    switch (state) {
+      case 'connecting':
+        // case 'thinking':
+        interval /= barCount;
+        break;
+
+      default:
+        break;
+    }
+  }
+  return interval;
+};
+/**
+ * @beta
+ */
+export interface BarVisualizerProps {
+  /** If set, the visualizer will transition between different voice assistant states */
+  state?: AgentState;
+  /** Number of bars that show up in the visualizer */
+  barCount?: number;
+  trackRef?: TrackReferenceOrPlaceholder;
+  options?: BarVisualizerOptions;
+  /**
+   * Custom React Native styles for the container.
+   */
+  style?: ViewStyle;
+}
+
+/**
+ * Visualizes audio signals from a TrackReference as bars.
+ * If the `state` prop is set, it automatically transitions between VoiceAssistant states.
+ * @beta
+ *
+ * @remarks For VoiceAssistant state transitions this component requires a voice assistant agent running with livekit-agents \>= 0.9.0
+ *
+ * @example
+ * ```tsx
+ * function SimpleVoiceAssistant() {
+ *   const { state, audioTrack } = useVoiceAssistant();
+ *   return (
+ *    <BarVisualizer
+ *      state={state}
+ *      trackRef={audioTrack}
+ *    />
+ *   );
+ * }
+ * ```
+ */
+export const BarVisualizer = ({
+  style = {},
+  state,
+  barCount = 5,
+  trackRef,
+  options,
+}: BarVisualizerProps) => {
+  let trackReference = useMaybeTrackRefContext();
+
+  if (trackRef) {
+    trackReference = trackRef;
+  }
+
+  const opacityAnimations = useRef<Animated.Value[]>([]).current;
+  let magnitudes = useMultibandTrackVolume(trackReference, { bands: barCount });
+
+  let opts = { ...defaultBarOptions, ...options };
+
+  const highlightedIndices = useBarAnimator(
+    state,
+    barCount,
+    getSequencerInterval(state, barCount) ?? 100
+  );
+
+  useEffect(() => {
+    let animations = [];
+    for (let i = 0; i < barCount; i++) {
+      if (!opacityAnimations[i]) {
+        opacityAnimations[i] = new Animated.Value(0.3);
+      }
+      let targetOpacity = 0.3;
+      if (highlightedIndices.includes(i)) {
+        targetOpacity = 1;
+      }
+      animations.push(
+        Animated.timing(opacityAnimations[i], {
+          toValue: targetOpacity,
+          duration: 250,
+          useNativeDriver: true,
+        })
+      );
+    }
+
+    let parallel = Animated.parallel(animations);
+    parallel.start();
+    return () => {
+      parallel.stop();
+    };
+  }, [highlightedIndices, barCount, opacityAnimations]);
+
+  let bars: React.ReactNode[] = [];
+  magnitudes.forEach((value, index) => {
+    let coerced = Math.min(opts.maxHeight, Math.max(opts.minHeight, value));
+    let coercedPercent = Math.min(100, Math.max(0, coerced * 100 + 5));
+    let opacity = opacityAnimations[index] ?? new Animated.Value(0.3);
+    let barStyle = {
+      opacity: opacity,
+      backgroundColor: opts.barColor,
+      borderRadius: opts.barBorderRadius,
+      width: opts.barWidth,
+    };
+    bars.push(
+      <Animated.View
+        key={index}
+        style={[
+          { height: `${coercedPercent}%` },
+          barStyle,
+          styles.volumeIndicator,
+        ]}
+      />
+    );
+  });
+
+  return <View style={{ ...style, ...styles.container }}>{bars}</View>;
+};
+const styles = StyleSheet.create({
+  container: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    justifyContent: 'space-evenly',
+  },
+  volumeIndicator: {
+    borderRadius: 12,
+  },
+});
+
+export const useBarAnimator = (
+  state: AgentState | undefined,
+  columns: number,
+  interval: number
+): number[] => {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<number[][]>([[]]);
+
+  useEffect(() => {
+    if (state === 'thinking') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === 'connecting' || state === 'initializing') {
+      const seq = [...generateConnectingSequenceBar(columns)];
+      setSequence(seq);
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === undefined) {
+      // highlight everything
+      setSequence([new Array(columns).fill(0).map((_, idx) => idx)]);
+    } else {
+      setSequence([[]]);
+    }
+    setIndex(0);
+  }, [state, columns]);
+
+  const animationFrameId = useRef<number | null>(null);
+  useEffect(() => {
+    let startTime = performance.now();
+
+    const animate = (time: number) => {
+      const timeElapsed = time - startTime;
+
+      if (timeElapsed >= interval) {
+        setIndex((prev) => prev + 1);
+        startTime = time;
+      }
+
+      animationFrameId.current = requestAnimationFrame(animate);
+    };
+
+    animationFrameId.current = requestAnimationFrame(animate);
+
+    return () => {
+      if (animationFrameId.current !== null) {
+        cancelAnimationFrame(animationFrameId.current);
+      }
+    };
+  }, [interval, columns, state, sequence.length]);
+
+  return sequence[index % sequence.length];
+};
+
+const generateListeningSequenceBar = (columns: number): number[][] => {
+  const center = Math.floor(columns / 2);
+  const noIndex = -1;
+
+  return [[center], [noIndex]];
+};
+
+const generateConnectingSequenceBar = (columns: number): number[][] => {
+  const seq: number[][] = [[]];
+
+  for (let x = 0; x < columns; x++) {
+    seq.push([x, columns - 1 - x]);
+  }
+
+  return seq;
+};

package/src/events/EventEmitter.ts

@@ -0,0 +1,51 @@
+import { NativeEventEmitter, type EmitterSubscription } from 'react-native';
+// @ts-ignore
+import EventEmitter from 'react-native/Libraries/vendor/emitter/EventEmitter';
+import LiveKitModule from '../LKNativeModule';
+
+// This emitter is going to be used to listen to all the native events (once) and then
+// re-emit them on a JS-only emitter.
+const nativeEmitter = new NativeEventEmitter(LiveKitModule);
+
+const NATIVE_EVENTS = ['LK_VOLUME_PROCESSED', 'LK_MULTIBAND_PROCESSED'];
+
+const eventEmitter = new EventEmitter();
+
+export function setupNativeEvents() {
+  for (const eventName of NATIVE_EVENTS) {
+    nativeEmitter.addListener(eventName, (...args) => {
+      eventEmitter.emit(eventName, ...args);
+    });
+  }
+}
+
+type EventHandler = (event: unknown) => void;
+type Listener = unknown;
+
+const _subscriptions: Map<Listener, EmitterSubscription[]> = new Map();
+
+export function addListener(
+  listener: Listener,
+  eventName: string,
+  eventHandler: EventHandler
+): void {
+  if (!NATIVE_EVENTS.includes(eventName)) {
+    throw new Error(`Invalid event: ${eventName}`);
+  }
+
+  if (!_subscriptions.has(listener)) {
+    _subscriptions.set(listener, []);
+  }
+
+  _subscriptions
+    .get(listener)
+    ?.push(eventEmitter.addListener(eventName, eventHandler));
+}
+
+export function removeListener(listener: Listener): void {
+  _subscriptions.get(listener)?.forEach((sub) => {
+    sub.remove();
+  });
+
+  _subscriptions.delete(listener);
+}

package/src/hooks/useMultibandTrackVolume.ts

@@ -0,0 +1,97 @@
+import { type TrackReferenceOrPlaceholder } from '@livekit/components-react';
+import {
+  Track,
+  type LocalAudioTrack,
+  type RemoteAudioTrack,
+} from 'livekit-client';
+import { useEffect, useMemo, useState } from 'react';
+import { addListener, removeListener } from '../events/EventEmitter';
+import LiveKitModule from '../LKNativeModule';
+
+/**
+ * Interface for configuring options for the useMultibandTrackVolume hook.
+ * @alpha
+ */
+export interface MultiBandTrackVolumeOptions {
+  /**
+   * the number of bands to split the audio into
+   */
+  bands?: number;
+  /**
+   * cut off frequency on the lower end
+   */
+  minFrequency?: number;
+  /**
+   * cut off frequency on the higher end
+   */
+  maxFrequency?: number;
+  /**
+   * update should run every x ms
+   */
+  updateInterval?: number;
+}
+
+const multibandDefaults = {
+  bands: 5,
+  minFrequency: 1000,
+  maxFrequency: 8000,
+  updateInterval: 40,
+} as const satisfies MultiBandTrackVolumeOptions;
+
+/**
+ * A hook for tracking the volume of an audio track across multiple frequency bands.
+ *
+ * @param trackOrTrackReference
+ * @returns A number array containing the volume for each frequency band.
+ */
+export function useMultibandTrackVolume(
+  trackOrTrackReference?:
+    | LocalAudioTrack
+    | RemoteAudioTrack
+    | TrackReferenceOrPlaceholder,
+  options: MultiBandTrackVolumeOptions = {}
+) {
+  const track =
+    trackOrTrackReference instanceof Track
+      ? trackOrTrackReference
+      : <LocalAudioTrack | RemoteAudioTrack | undefined>(
+          trackOrTrackReference?.publication?.track
+        );
+  const opts = useMemo(() => {
+    return { ...multibandDefaults, ...options };
+  }, [options]);
+  const mediaStreamTrack = track?.mediaStreamTrack;
+
+  let [magnitudes, setMagnitudes] = useState<number[]>([]);
+  useEffect(() => {
+    let listener = Object();
+    let reactTag: string | null = null;
+    if (mediaStreamTrack) {
+      reactTag = LiveKitModule.createMultibandVolumeProcessor(
+        opts,
+        mediaStreamTrack._peerConnectionId ?? -1,
+        mediaStreamTrack.id
+      );
+      addListener(listener, 'LK_MULTIBAND_PROCESSED', (event: any) => {
+        if (event.magnitudes && reactTag && event.id === reactTag) {
+          console.log('event received: ', event.magnitudes[0]);
+          setMagnitudes(event.magnitudes);
+        }
+      });
+    }
+    return () => {
+      if (mediaStreamTrack) {
+        removeListener(listener);
+        if (reactTag) {
+          LiveKitModule.deleteMultibandVolumeProcessor(
+            reactTag,
+            mediaStreamTrack._peerConnectionId ?? -1,
+            mediaStreamTrack.id
+          );
+        }
+      }
+    };
+  }, [mediaStreamTrack, opts]);
+
+  return magnitudes;
+}

package/src/hooks/useTrackVolume.ts

@@ -0,0 +1,62 @@
+import { type TrackReferenceOrPlaceholder } from '@livekit/components-react';
+import {
+  Track,
+  type LocalAudioTrack,
+  type RemoteAudioTrack,
+} from 'livekit-client';
+import { useEffect, useState } from 'react';
+import { addListener, removeListener } from '../events/EventEmitter';
+import LiveKitModule from '../LKNativeModule';
+
+/**
+ * A hook for tracking the volume of an audio track.
+ *
+ * @param trackOrTrackReference
+ * @returns A number between 0-1 representing the volume.
+ */
+export function useTrackVolume(
+  trackOrTrackReference?:
+    | LocalAudioTrack
+    | RemoteAudioTrack
+    | TrackReferenceOrPlaceholder
+) {
+  const track =
+    trackOrTrackReference instanceof Track
+      ? trackOrTrackReference
+      : <LocalAudioTrack | RemoteAudioTrack | undefined>(
+          trackOrTrackReference?.publication?.track
+        );
+
+  const mediaStreamTrack = track?.mediaStreamTrack;
+
+  let [volume, setVolume] = useState(0.0);
+  useEffect(() => {
+    let listener = Object();
+    let reactTag: string | null = null;
+    if (mediaStreamTrack) {
+      reactTag = LiveKitModule.createVolumeProcessor(
+        mediaStreamTrack._peerConnectionId ?? -1,
+        mediaStreamTrack.id
+      );
+      addListener(listener, 'LK_VOLUME_PROCESSED', (event: any) => {
+        if (event.volume && reactTag && event.id === reactTag) {
+          setVolume(event.volume);
+        }
+      });
+    }
+    return () => {
+      if (mediaStreamTrack) {
+        removeListener(listener);
+        if (reactTag) {
+          LiveKitModule.deleteVolumeProcessor(
+            reactTag,
+            mediaStreamTrack._peerConnectionId ?? -1,
+            mediaStreamTrack.id
+          );
+        }
+      }
+    };
+  }, [mediaStreamTrack]);
+
+  return volume;
+}

package/src/hooks.ts

@@ -42,4 +42,6 @@ export type {
 
 export type { ReceivedDataMessage } from '@livekit/components-core';
 export * from './hooks/useE2EEManager';
+export * from './hooks/useTrackVolume';
+export * from './hooks/useMultibandTrackVolume';
 export type { UseRNE2EEManagerOptions } from './hooks/useE2EEManager';

package/src/index.tsx

@@ -17,6 +17,7 @@ import { type LiveKitReactNativeInfo } from 'livekit-client';
 import type { LogLevel, SetLogLevelOptions } from './logger';
 import RNE2EEManager from './e2ee/RNE2EEManager';
 import RNKeyProvider, { type RNKeyProviderOptions } from './e2ee/RNKeyProvider';
+import { setupNativeEvents } from './events/EventEmitter';
 
 /**
  * Registers the required globals needed for LiveKit to work.
@@ -33,6 +34,7 @@ export function registerGlobals() {
   shimArrayAt();
   shimAsyncIterator();
   shimIterator();
+  setupNativeEvents();
 }
 
 /**
@@ -99,6 +101,7 @@ function shimIterator() {
   shim();
 }
 export * from './hooks';
+export * from './components/BarVisualizer';
 export * from './components/LiveKitRoom';
 export * from './components/VideoTrack';
 export * from './components/VideoView'; // deprecated