agora-appbuilder-core 4.0.35 → 4.1.0-beta-2

package/template/src/ai-agent/components/mobile/MobileLayoutComponent.tsx

@@ -0,0 +1,106 @@
+import React, {useContext, useEffect, useState} from 'react';
+import {View} from 'react-native';
+import {ILocalAudioTrack, IRemoteAudioTrack} from 'agora-rtc-sdk-ng';
+import {
+  MaxVideoView,
+  useContent,
+  useLocalUid,
+  LayoutComponent,
+  useRtc,
+  useLocalAudio,
+  useIsAudioEnabled,
+} from 'customization-api';
+import AudioVisualizer, {DisconnectedView} from '../AudioVisualizer';
+import {ActiveSpeakerAnimation} from '../../components/LocalAudioWave';
+import {AgentContext} from '../AgentControls/AgentContext';
+
+const MobileLayoutComponent: LayoutComponent = () => {
+  const localUid = useLocalUid();
+  const {defaultContent, activeUids} = useContent();
+  const {RtcEngineUnsafe} = useRtc();
+  const [localTracks, setLocalTrack] = useState<ILocalAudioTrack | null>(null);
+  const [remoteTrack, setRemoteTrack] = useState<IRemoteAudioTrack | null>(
+    null,
+  );
+  const {agentUID} = useContext(AgentContext);
+
+  const {getLocalAudioStream, getRemoteAudioStream} = useLocalAudio();
+  const isAudioEnabled = useIsAudioEnabled();
+  const connected = activeUids.includes(agentUID);
+  console.log({activeUids}, 'active uids');
+
+  useEffect(() => {
+    if (getLocalAudioStream()) {
+      setLocalTrack(getLocalAudioStream());
+    }
+  }, [RtcEngineUnsafe]);
+
+  useEffect(() => {}, [isAudioEnabled]);
+
+  useEffect(() => {
+    if (getRemoteAudioStream(agentUID)) {
+      setRemoteTrack(getRemoteAudioStream(agentUID));
+    }
+  }, [activeUids]);
+
+  return (
+    <View
+      style={{
+        flex: 1,
+        display: 'flex',
+        flexDirection: 'column',
+      }}>
+      <MaxVideoView
+        containerStyle={{borderRadius: 12, borderWidth: 0}}
+        innerContainerStyle={{backgroundColor: '#222'}}
+        user={{
+          ...defaultContent[agentUID],
+          name: 'OpenAI',
+          video: false,
+        }}
+        CustomChild={() =>
+          connected ? (
+            <AudioVisualizer audioTrack={remoteTrack} />
+          ) : (
+            <DisconnectedView isConnected={connected} />
+          )
+        }
+        hideMenuOptions={true}
+      />
+      <View
+        style={{
+          position: 'absolute',
+          top: 8,
+          right: 8,
+          display: 'flex',
+          flexDirection: 'row',
+          height: 100,
+          width: 215,
+        }}>
+        <MaxVideoView
+          containerStyle={{borderRadius: 8, borderWidth: 0}}
+          user={defaultContent[localUid]}
+          avatarRadius={48}
+          hideMenuOptions={true}
+        />
+        <View
+          style={{
+            position: 'absolute',
+            width: 100,
+            height: 50,
+            bottom: 16,
+            right: -38,
+          }}>
+          {localTracks && isAudioEnabled(localUid) && (
+            <ActiveSpeakerAnimation
+              audioTrack={localTracks}
+              isMuted={!isAudioEnabled(localUid)}
+            />
+          )}
+        </View>
+      </View>
+    </View>
+  );
+};
+
+export default MobileLayoutComponent;

package/template/src/ai-agent/components/mobile/Topbar.tsx

@@ -0,0 +1,62 @@
+import React from 'react';
+import {View, StyleSheet, Text, TouchableOpacity} from 'react-native';
+import {LogoIcon, SettingsIcon} from '../icons';
+import {SidePanelType, useSidePanel} from 'customization-api';
+import ThemeConfig from '../../../theme';
+
+const MobileTopbar = () => {
+  const {setSidePanel} = useSidePanel();
+  return (
+    <View style={style.rootStyle}>
+      <View style={style.containerStyle}>
+        <View style={style.logContainerStyle}>
+          <LogoIcon />
+        </View>
+        <Text style={style.textStyle}>Agora & OpenAI Demo</Text>
+      </View>
+      <TouchableOpacity
+        onPress={() => {
+          setSidePanel(SidePanelType.Settings);
+        }}>
+        <SettingsIcon />
+      </TouchableOpacity>
+    </View>
+  );
+};
+
+const style = StyleSheet.create({
+  rootStyle: {
+    display: 'flex',
+    flexDirection: 'row',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    paddingBottom: 20,
+  },
+  containerStyle: {
+    display: 'flex',
+    flexDirection: 'row',
+    justifyContent: 'center',
+    alignItems: 'center',
+    gap: 4,
+    alignSelf: 'stretch',
+  },
+  logContainerStyle: {
+    display: 'flex',
+    width: 40,
+    height: 40,
+    paddingRight: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  textStyle: {
+    color: '#C3C3C3',
+    textAlign: 'center',
+    fontFamily: ThemeConfig.FontFamily.sansPro,
+    fontSize: 18,
+    fontStyle: 'normal',
+    fontWeight: '600',
+    lineHeight: 18,
+  },
+});
+
+export default MobileTopbar;

package/template/src/ai-agent/components/react-audio-visualize/LiveAudioVisualizer/LiveAudioVisualizer.tsx

@@ -0,0 +1,173 @@
+import React, {
+  type ReactElement,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from 'react';
+import {calculateBarData, draw} from './utils';
+import {ILocalAudioTrack, IRemoteAudioTrack} from 'agora-rtc-sdk-ng';
+
+export interface Props {
+  /**
+   * Width of the visualization. Default: "100%"
+   */
+  width?: number | string;
+  /**
+   * Height of the visualization. Default: "100%"
+   */
+  height?: number | string;
+  /**
+   * Width of each individual bar in the visualization. Default: `2`
+   */
+  barWidth?: number;
+  /**
+   * Gap between each bar in the visualization. Default: `1`
+   */
+  gap?: number;
+  /**
+   * BackgroundColor for the visualization: Default: `transparent`
+   */
+  backgroundColor?: string;
+  /**
+   * Color of the bars drawn in the visualization. Default: `"rgb(160, 198, 255)"`
+   */
+  barColor?: string;
+  /**
+   * An unsigned integer, representing the window size of the FFT, given in number of samples.
+   * Default: `1024`
+   */
+  fftSize?:
+    | 32
+    | 64
+    | 128
+    | 256
+    | 512
+    | 1024
+    | 2048
+    | 4096
+    | 8192
+    | 16384
+    | 32768;
+  /**
+   * A double, representing the maximum decibel value for scaling the FFT analysis data
+   * Default: `-10`
+   */
+  maxDecibels?: number;
+  /**
+   * A double, representing the minimum decibel value for scaling the FFT analysis data
+   * Default: `-90`
+   */
+  minDecibels?: number;
+  /**
+   * A double within the range 0 to 1 (0 meaning no time averaging). The default value is 0.8.
+   * Default: `0.4`
+   */
+  smoothingTimeConstant?: number;
+  audioTrack?: ILocalAudioTrack | IRemoteAudioTrack;
+}
+
+const LiveAudioVisualizer: (props: Props) => ReactElement = ({
+  width = '100%',
+  height = '100%',
+  barWidth = 2,
+  gap = 1,
+  backgroundColor = 'transparent',
+  barColor = 'rgb(160, 198, 255)',
+  fftSize = 1024,
+  maxDecibels = -10,
+  minDecibels = -90,
+  smoothingTimeConstant = 0.4,
+  audioTrack,
+}: Props) => {
+  const [audioContext, setAudioContext] = useState<AudioContext | null>(null);
+  const [audioSource, setAudioSource] = useState<MediaStreamAudioSourceNode>();
+  const [analyser, setAnalyser] = useState<AnalyserNode>();
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+
+  useEffect(() => {
+    if (!audioTrack) {
+      // Clean up if track is null
+      if (audioContext) {
+        audioContext.state !== 'closed' && audioContext.close();
+        setAudioContext(null);
+        setAnalyser(null);
+        setAudioSource(null);
+      }
+      return;
+    }
+
+    const newAudioContext = new (window.AudioContext ||
+      (window as any).webkitAudioContext)();
+    const newAnalyser = newAudioContext.createAnalyser();
+    newAnalyser.fftSize = fftSize;
+    newAnalyser.minDecibels = minDecibels;
+    newAnalyser.maxDecibels = maxDecibels;
+    newAnalyser.smoothingTimeConstant = smoothingTimeConstant;
+
+    const source = newAudioContext.createMediaStreamSource(
+      new MediaStream([audioTrack.getMediaStreamTrack()]),
+    );
+    source.connect(newAnalyser);
+
+    setAudioContext(newAudioContext);
+    setAnalyser(newAnalyser);
+    setAudioSource(source);
+
+    return () => {
+      source.disconnect();
+      newAnalyser.disconnect();
+      newAudioContext.state !== 'closed' && newAudioContext.close();
+    };
+  }, [audioTrack, fftSize, minDecibels, maxDecibels, smoothingTimeConstant]);
+
+  useEffect(() => {
+    if (analyser && audioTrack) {
+      requestAnimationFrame(report);
+    }
+  }, [analyser, audioTrack]);
+
+  const report = useCallback(() => {
+    if (!analyser || !audioContext || !audioTrack) return;
+
+    const data = new Uint8Array(analyser.frequencyBinCount);
+    analyser.getByteFrequencyData(data);
+    processFrequencyData(data);
+
+    if (audioTrack.isPlaying) {
+      requestAnimationFrame(report);
+    }
+  }, [analyser, audioContext, audioTrack]);
+
+  const processFrequencyData = (data: Uint8Array): void => {
+    if (!canvasRef.current) return;
+
+    const dataPoints = calculateBarData(
+      data,
+      canvasRef.current.width,
+      barWidth,
+      gap,
+    );
+    draw(
+      dataPoints,
+      canvasRef.current,
+      barWidth,
+      gap,
+      backgroundColor,
+      barColor,
+    );
+  };
+
+  return (
+    <canvas
+      ref={canvasRef}
+      width={width}
+      height={height}
+      style={{
+        aspectRatio: 'unset',
+      }}
+    />
+  );
+};
+
+export {LiveAudioVisualizer};

package/template/src/ai-agent/components/react-audio-visualize/LiveAudioVisualizer/index.ts

@@ -0,0 +1 @@
+export {LiveAudioVisualizer} from './LiveAudioVisualizer';

package/template/src/ai-agent/components/react-audio-visualize/LiveAudioVisualizer/utils.ts

@@ -0,0 +1,102 @@
+interface CustomCanvasRenderingContext2D extends CanvasRenderingContext2D {
+  roundRect: (
+    x: number,
+    y: number,
+    w: number,
+    h: number,
+    radius: number,
+  ) => void;
+}
+
+export const calculateBarData = (
+  frequencyData: Uint8Array,
+  width: number,
+  barWidth: number,
+  gap: number,
+): number[] => {
+  let units = width / (barWidth + gap);
+  let step = Math.floor(frequencyData.length / units);
+
+  if (units > frequencyData.length) {
+    units = frequencyData.length;
+    step = 1;
+  }
+
+  const data: number[] = [];
+
+  for (let i = 0; i < units; i++) {
+    let sum = 0;
+
+    for (let j = 0; j < step && i * step + j < frequencyData.length; j++) {
+      sum += frequencyData[i * step + j];
+    }
+    data.push(sum / step);
+  }
+  return data;
+};
+
+export const draw = (
+  data: number[],
+  canvas: HTMLCanvasElement,
+  barWidth: number,
+  gap: number,
+  backgroundColor: string,
+  barColor: string,
+): void => {
+  const amp = canvas.height / 2;
+
+  const ctx = canvas.getContext('2d') as CustomCanvasRenderingContext2D;
+  if (!ctx) return;
+
+  ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+  if (backgroundColor !== 'transparent') {
+    ctx.fillStyle = backgroundColor;
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+  }
+
+  // const sampled = data.slice(0, 8);
+  //
+  // const downSampled = new Array(5).fill(0);
+  //
+  // const downPoint = Math.floor(sampled.length / 5);
+  //
+  // sampled.forEach((dp, i) => {
+  // 	downSampled[Math.floor(i / downPoint)] += dp;
+  // });
+
+  // const sampled = new Array(Math.ceil(data.length / 5)).fill(0);
+
+  // for (const dataPoint of data) {
+  // 	const index = Math.floor(dataPoint / 5);
+  // 	sampled[index] += dataPoint;
+  // }
+
+  const displayData = data.slice(0, 5);
+
+  const newBarWidth =
+    (canvas.width - gap * (displayData.length - 1)) / displayData.length;
+
+  const minHeight = newBarWidth;
+
+  displayData.forEach((dpo, i) => {
+    const dp = dpo;
+
+    ctx.fillStyle = barColor;
+
+    const x = i * (newBarWidth + gap);
+    const y = amp - dp / 2;
+    const w = newBarWidth;
+    const h = dp || 1;
+
+    ctx.beginPath();
+    if (ctx.roundRect) {
+      // making sure roundRect is supported by the browser
+      ctx.roundRect(x, y - minHeight / 2, w, h + minHeight, w / 2);
+      ctx.fill();
+    } else {
+      // fallback for browsers that do not support roundRect
+      ctx.fillRect(x, y, w, h);
+    }
+  });
+};

package/template/src/ai-agent/components/react-audio-visualize/index.ts

@@ -0,0 +1 @@
+export {LiveAudioVisualizer} from './LiveAudioVisualizer';

package/template/src/ai-agent/components/utils.ts

@@ -0,0 +1,15 @@
+import {
+  AGENT_PROXY_URL,
+  AGORA_SSO_LOGIN_PATH,
+  AGORA_SSO_CLIENT_ID,
+  AGORA_SSO_BASE,
+} from '../components/AgentControls/const';
+
+export const handleSSOLogin = () => {
+  const REDIRECT_URL = `${AGENT_PROXY_URL}/login`;
+  const originURL = window.location.origin + '/validate';
+
+  const ssoUrl = `${AGORA_SSO_BASE}/${AGORA_SSO_LOGIN_PATH}?scope=basic_info&response_type=code&state=url=${originURL}&redirect_uri=${REDIRECT_URL}&client_id=${AGORA_SSO_CLIENT_ID}`;
+  console.log('sso', ssoUrl);
+  window.location.href = ssoUrl;
+};