@trtc/calls-uikit-react 4.4.3-beta.2 → 4.4.4

package/src/Components/hooks/useAIAssistant.ts

@@ -0,0 +1,174 @@
+/**
+ * AI Assistant Hook
+ * @description React hook for AI Transcriber functionality
+ */
+import { useState, useEffect, useCallback } from 'react';
+import { TUIStore, StoreName, NAME } from '../../TUICallService/index';
+
+// Translation content interface
+interface ITranslationContent {
+  language: string;
+  text: string;
+}
+
+// Transcriber message interface (simplified for React version)
+interface TranscriberMessage {
+  segmentId: string;
+  speakerUserId: string;
+  sourceText: string;
+  isCompleted: boolean;
+  translationTexts: any;
+}
+
+// Converted message interface
+interface ITranslationInfo {
+  roundId: string;
+  sender: string;
+  nick?: string;
+  text: string;
+  end: boolean;
+  translation: ITranslationContent[];
+}
+
+/**
+ * AI Assistant hook
+ * Provides state management for AI transcription functionality
+ */
+export function useAIAssistant() {
+  // AI Transcriber Status State
+  const [isAITranscriberEnabled, setIsAITranscriberEnabled] = useState<boolean>(false);
+  const [isAITranscriberRunning, setIsAITranscriberRunning] = useState<boolean>(false);
+  
+  // Converted message list
+  const [subtitleInfoList, setSubtitleInfoList] = useState<ITranslationInfo[]>([]);
+
+  // Helper functions for message conversion
+  const parseTranslationTexts = useCallback((translationTexts: any): ITranslationContent[] => {
+    if (!translationTexts) {
+      return [];
+    }
+
+    // Handle Map type
+    if (translationTexts instanceof Map) {
+      return Array.from(translationTexts.entries()).map(([language, content]) => ({
+        language: String(language),
+        text: ensureStringContent(content)
+      }));
+    }
+
+    // Handle plain object type
+    if (typeof translationTexts === 'object') {
+      return Object.entries(translationTexts).map(([language, content]) => ({
+        language: String(language),
+        text: ensureStringContent(content)
+      }));
+    }
+
+    return [];
+  }, []);
+
+  const ensureStringContent = useCallback((content: any): string => {
+    if (content === null || content === undefined) {
+      return '';
+    }
+    
+    if (typeof content === 'string') {
+      return content;
+    }
+    
+    if (typeof content === 'object') {
+      try {
+        return JSON.stringify(content);
+      } catch (error) {
+        console.warn('Failed to stringify translation content:', error);
+        return '[Invalid Object]';
+      }
+    }
+    
+    return String(content);
+  }, []);
+
+  const convertToLegacyFormat = useCallback((messages: TranscriberMessage[]): ITranslationInfo[] => {
+    return messages.map(msg => ({
+      roundId: msg.segmentId,
+      sender: msg.speakerUserId,
+      nick: '',
+      text: msg.sourceText,
+      end: msg.isCompleted,
+      translation: parseTranslationTexts(msg.translationTexts)
+    }));
+  }, [parseTranslationTexts]);
+
+  // Store watcher callbacks
+  const handleAITranscriberEnabledChange = useCallback((value: boolean) => {
+    setIsAITranscriberEnabled(value || false);
+  }, []);
+
+  const handleAITranscriberRunningChange = useCallback((value: boolean) => {
+    setIsAITranscriberRunning(value || false);
+  }, []);
+
+  const handleRealtimeMessageListChange = useCallback((messageList: TranscriberMessage[]) => { 
+    if (messageList && messageList.length > 0) {
+      const convertedMessages = convertToLegacyFormat(messageList);
+      setSubtitleInfoList(convertedMessages);
+    } else {
+      setSubtitleInfoList([]);
+    }
+  }, [convertToLegacyFormat]);
+
+  // Effect for lifecycle management
+  useEffect(() => {
+    // Try to get initial values from TUIStore if available
+    try {
+      const initialEnabled = TUIStore.getData(StoreName.CALL, NAME.IS_AI_TRANSCRIBER_ENABLED);
+      const initialRunning = TUIStore.getData(StoreName.CALL, NAME.IS_AI_TRANSCRIBER_RUNNING);
+      const initialMessages = TUIStore.getData(StoreName.CALL, NAME.REALTIME_MESSAGE_LIST);
+      
+      if (initialEnabled !== undefined) {
+        setIsAITranscriberEnabled(initialEnabled);
+      }
+      if (initialRunning !== undefined) {
+        setIsAITranscriberRunning(initialRunning);
+      }
+      if (initialMessages) {
+        handleRealtimeMessageListChange(initialMessages);
+      }
+
+      // Subscribe to TUIStore changes if the store supports these fields
+      if (NAME.IS_AI_TRANSCRIBER_ENABLED && NAME.IS_AI_TRANSCRIBER_RUNNING && NAME.REALTIME_MESSAGE_LIST) {
+        TUIStore.watch(StoreName.CALL, {
+          [NAME.IS_AI_TRANSCRIBER_ENABLED]: handleAITranscriberEnabledChange,
+          [NAME.IS_AI_TRANSCRIBER_RUNNING]: handleAITranscriberRunningChange,
+          [NAME.REALTIME_MESSAGE_LIST]: handleRealtimeMessageListChange,
+        });
+      }
+    } catch (error) {
+      console.warn('AI Assistant hook initialization failed:', error);
+      // Set default values for development/testing
+      setIsAITranscriberEnabled(true);
+      setIsAITranscriberRunning(false);
+    }
+
+    // Cleanup function
+    return () => {
+      try {
+        if (NAME.IS_AI_TRANSCRIBER_ENABLED && NAME.IS_AI_TRANSCRIBER_RUNNING && NAME.REALTIME_MESSAGE_LIST) {
+          TUIStore.unwatch(StoreName.CALL, {
+            [NAME.IS_AI_TRANSCRIBER_ENABLED]: handleAITranscriberEnabledChange,
+            [NAME.IS_AI_TRANSCRIBER_RUNNING]: handleAITranscriberRunningChange,
+            [NAME.REALTIME_MESSAGE_LIST]: handleRealtimeMessageListChange,
+          });
+        }
+      } catch (error) {
+        console.warn('AI Assistant hook cleanup failed:', error);
+      }
+    };
+  }, [handleAITranscriberEnabledChange, handleAITranscriberRunningChange, handleRealtimeMessageListChange]);
+
+  return {
+    isAITranscriberEnabled,
+    isAITranscriberRunning,
+    subtitleInfoList,
+  };
+}

package/src/TUICallService/CallService/AIAssistant.ts

@@ -1,51 +1,297 @@
-// @ts-ignore
-import { aiAssistant } from '../../Components/components/common/AIAssistant/index';
-import { getRemoteUserProfile } from './utils';
-
-interface IAIAssistant {
-  enableAISubtitle: (enable: boolean) => void;
-  setEngineInstance: (engineInstance: any) => void;
-  setImInstance: (imInstance: any) => void;
-};
-
-export class AIAssistant implements IAIAssistant {
-  static instance: IAIAssistant;
-  private _tuiCallEngine: any = null;
-  private _imInstance: any = null;
-
-  static getInstance() {
+/**
+ * AI Realtime Transcriber State Management
+ * @module AITranscriberState
+ * @description Manages realtime voice transcription state and operations, including message reception and history
+ */
+import { ITUIStore } from '../interface/ITUIStore';
+import { StoreName, NAME } from '../const/index';
+import TuiStore from '../TUIStore/tuiStore';
+
+// AI Transcriber events
+export enum RealtimeTranscriberEvent {
+  onReceiveTranscriberMessage = 'onReceiveTranscriberMessage',
+  onRealtimeTranscriberStarted = 'onRealtimeTranscriberStarted',
+  onRealtimeTranscriberStopped = 'onRealtimeTranscriberStopped',
+  onRealtimeTranscriberError = 'onRealtimeTranscriberError',
+}
+
+// Transcriber language type
+export type TranscriberLanguage = string;
+
+// Transcriber message interface
+export interface TranscriberMessage {
+  segmentId: string;
+  speakerUserId: string;
+  sourceText: string;
+  translationTexts?: Map<string, string>;
+  timestamp: number;
+  isCompleted: boolean;
+}
+
+// Event callback types
+export interface RealtimeTranscriberEventInfoMap {
+  [RealtimeTranscriberEvent.onReceiveTranscriberMessage]: {
+    roomId: string | number;
+    message: TranscriberMessage;
+  };
+  [RealtimeTranscriberEvent.onRealtimeTranscriberStarted]: {
+    roomId: string | number;
+    transcriberRobotId: string;
+    sourceLanguage: string;
+  };
+  [RealtimeTranscriberEvent.onRealtimeTranscriberStopped]: {
+    roomId: string | number;
+    transcriberRobotId: string;
+  };
+  [RealtimeTranscriberEvent.onRealtimeTranscriberError]: {
+    roomId: string | number;
+    transcriberRobotId: string;
+    error: number;
+    errorMessage: string;
+  };
+}
+
+export type RealtimeTranscriberEventCallback = <T extends RealtimeTranscriberEvent>(
+  eventInfo: RealtimeTranscriberEventInfoMap[T]
+) => void;
+
+const TUIStore: ITUIStore = TuiStore.getInstance();
+
+export default class AIAssistant {
+  static instance: AIAssistant;
+  private _callService: any;
+  private eventListeners = new Map<RealtimeTranscriberEvent, RealtimeTranscriberEventCallback[]>();
+
+  constructor(options: { callService: any }) {
+    this._callService = options.callService;
+  }
+
+  static getInstance(options: { callService: any }) {
     if (!AIAssistant.instance) {
-      AIAssistant.instance = new AIAssistant();
+      AIAssistant.instance = new AIAssistant(options);
     }
     return AIAssistant.instance;
   }
 
-  public enableAISubtitle(enable: boolean): void {
-    this._tuiCallEngine?.reportLog?.({
-      name: 'TUICallKit.enableAISubtitle.start',
-      data: { enable },
-    });
+  /**
+   * Get AI Transcriber Manager instance
+   * @returns AITranscriberManager instance or null if not available
+   */
+  private getTranscriberManager = () => {
+    const callEngine = this._callService?.getTUICallEngineInstance();
+    if (!callEngine) {
+      console.warn(`${NAME.PREFIX}Failed to get call engine instance.`);
+      return null;
+    }
 
-    const trtcCloudInstance = this._tuiCallEngine.getTRTCCloudInstance();
-    const getNickName = async(userId: string) => {
-      const res = await getRemoteUserProfile([userId], this._imInstance);
-      return res.length ? res[0] : [];
+    const trtcCloud = callEngine.getTRTCCloudInstance?.();
+    if (!trtcCloud || !trtcCloud.getAITranscriberManager) {
+      console.warn(`${NAME.PREFIX}AI Transcriber not supported.`);
+      return null;
     }
-    if (enable) {
-      aiAssistant.initASR({
-        trtcCloudInstance,
-        // @ts-ignore
-        getNickName,
+
+    return trtcCloud.getAITranscriberManager();
+  };
+
+  private triggerEvent = <T extends RealtimeTranscriberEvent>(
+    event: T,
+    eventInfo: RealtimeTranscriberEventInfoMap[T],
+  ): void => {
+    const listeners = this.eventListeners.get(event);
+    if (listeners) {
+      listeners.forEach((callback) => {
+        try {
+          callback(eventInfo);
+        } catch (error) {
+          console.error(`${NAME.PREFIX}Error in event callback for ${event}:`, error);
+        }
       });
+    }
+  };
+
+  public startRealtimeTranscriber = async (config: {
+    sourceLanguage: TranscriberLanguage;
+    translationLanguages?: TranscriberLanguage[];
+  }) => {
+    const transcriberManager = this.getTranscriberManager();
+    if (!transcriberManager) {
+      return null;
+    }
+
+    try {
+      const transcriberRobotId = await transcriberManager.startRealtimeTranscriber(config);
+      
+      TUIStore.update(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID, transcriberRobotId);
+      TUIStore.update(StoreName.CALL, NAME.IS_AI_TRANSCRIBER_RUNNING, true);
+      return transcriberRobotId;
+    } catch (error) {
+      console.error(`${NAME.PREFIX}startRealtimeTranscriber error:`, error);
+      throw error;
+    }
+  };
+
+  public stopRealtimeTranscriber = async () => {
+    const transcriberManager = this.getTranscriberManager();
+    if (!transcriberManager) {
+      return;
+    }
+
+    try {
+      const transcriberRobotId = TUIStore.getData(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID);
+      
+      if (transcriberRobotId) {
+        await transcriberManager.stopRealtimeTranscriber(transcriberRobotId);
+      }
+      
+      TUIStore.update(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID, null);
+      TUIStore.update(StoreName.CALL, NAME.IS_AI_TRANSCRIBER_RUNNING, false);
+    } catch (error) {
+      console.error(`${NAME.PREFIX}stopRealtimeTranscriber error:`, error);
+      throw error;
+    }
+  };
+
+  public updateRealTimeTranscriber = async (config: {
+    sourceLanguage: TranscriberLanguage;
+    translationLanguages?: TranscriberLanguage[];
+  }) => {
+    const transcriberManager = this.getTranscriberManager();
+    if (!transcriberManager) {
+      return null;
+    }
+
+    try {
+      const currentTranscriberRobotId = TUIStore.getData(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID);
+      
+      if (currentTranscriberRobotId) {
+        await transcriberManager.stopRealtimeTranscriber(currentTranscriberRobotId);
+      }
+      
+      const newTranscriberRobotId = await transcriberManager.startRealtimeTranscriber(config);
+      TUIStore.update(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID, newTranscriberRobotId);
+      
+      return newTranscriberRobotId;
+    } catch (error) {
+      console.error(`${NAME.PREFIX}updateRealTimeTranscriber error:`, error);
+      throw error;
+    }
+  };
+
+  public subscribeEvent = (event: RealtimeTranscriberEvent, callback: RealtimeTranscriberEventCallback) => {
+    if (!this.eventListeners.has(event)) {
+      this.eventListeners.set(event, []);
+    }
+    this.eventListeners.get(event)?.push(callback);
+  };
+
+  public unsubscribeEvent = (event: RealtimeTranscriberEvent, callback: RealtimeTranscriberEventCallback) => {
+    const listeners = this.eventListeners.get(event);
+    if (listeners) {
+      const index = listeners.indexOf(callback);
+      if (index > -1) {
+        listeners.splice(index, 1);
+      }
+      if (listeners.length === 0) {
+        this.eventListeners.delete(event);
+      }
+    }
+  };
+
+  private onReceiveTranscriberMessage = (
+    _roomId: string | number,
+    message: {
+      segmentId: string;
+      speakerUserId: string;
+      sourceText: string;
+      translationTexts?: Map<string, string>;
+      timestamp: number;
+      isCompleted: boolean;
+    },
+  ) => {
+    const { segmentId } = message;
+    let realtimeMessageList: TranscriberMessage[] = TUIStore.getData(StoreName.CALL, NAME.REALTIME_MESSAGE_LIST) || [];
+    
+    const idx = realtimeMessageList.findIndex(msg => msg.segmentId === segmentId);
+    if (idx >= 0) {
+      realtimeMessageList[idx] = message;
     } else {
-      aiAssistant.destroyASR();
+      realtimeMessageList.push(message);
     }
-  }
+    
+    TUIStore.update(StoreName.CALL, NAME.REALTIME_MESSAGE_LIST, [...realtimeMessageList]);
+    
+    this.triggerEvent(RealtimeTranscriberEvent.onReceiveTranscriberMessage, {
+      roomId: _roomId,
+      message,
+    });
+  };
 
-  public setEngineInstance(engineInstance: any): void {
-    this._tuiCallEngine = engineInstance;
-  }
-  public setImInstance(imInstance: any): void {
-    this._imInstance = imInstance;
-  }
+  private onRealtimeTranscriberStarted = (_roomId: string | number, _transcriberRobotId: string, _sourceLanguage: string) => {
+    this.triggerEvent(RealtimeTranscriberEvent.onRealtimeTranscriberStarted, {
+      roomId: _roomId,
+      transcriberRobotId: _transcriberRobotId,
+      sourceLanguage: _sourceLanguage,
+    });
+  };
+
+  private onRealtimeTranscriberStopped = (_roomId: string | number, _transcriberRobotId: string) => {
+    this.triggerEvent(RealtimeTranscriberEvent.onRealtimeTranscriberStopped, {
+      roomId: _roomId,
+      transcriberRobotId: _transcriberRobotId,
+    });
+  };
+
+  private onRealtimeTranscriberError = (_roomId: string | number, _transcriberRobotId: string, error: number, errorMessage: string) => {
+    this.triggerEvent(RealtimeTranscriberEvent.onRealtimeTranscriberError, {
+      roomId: _roomId,
+      transcriberRobotId: _transcriberRobotId,
+      error,
+      errorMessage,
+    });
+  };
+
+  public bindEvent = () => {
+    const transcriberManager = this.getTranscriberManager();
+    if (!transcriberManager) {
+      return;
+    }
+
+    try {
+      transcriberManager.addListener({
+        onReceiveTranscriberMessage: this.onReceiveTranscriberMessage,
+        onRealtimeTranscriberStarted: this.onRealtimeTranscriberStarted,
+        onRealtimeTranscriberStopped: this.onRealtimeTranscriberStopped,
+        onRealtimeTranscriberError: this.onRealtimeTranscriberError,
+      });
+    } catch (error) {
+      console.error(`${NAME.PREFIX}bindEvent error:`, error);
+    }
+  };
+
+  public unbindEvent = () => {
+    const transcriberManager = this.getTranscriberManager();
+    if (!transcriberManager) {
+      return;
+    }
+
+    try {
+      transcriberManager.removeListener({
+        onReceiveTranscriberMessage: this.onReceiveTranscriberMessage,
+        onRealtimeTranscriberStarted: this.onRealtimeTranscriberStarted,
+        onRealtimeTranscriberStopped: this.onRealtimeTranscriberStopped,
+        onRealtimeTranscriberError: this.onRealtimeTranscriberError,
+      });
+    } catch (error) {
+      console.error(`${NAME.PREFIX}unbindEvent error:`, error);
+    }
+  };
+
+  public reset = () => {
+    TUIStore.update(StoreName.CALL, NAME.TRANSCRIBER_ROBOT_ID, null);
+    TUIStore.update(StoreName.CALL, NAME.REALTIME_MESSAGE_LIST, []);
+    TUIStore.update(StoreName.CALL, NAME.IS_AI_TRANSCRIBER_ENABLED, false);
+    this.eventListeners.clear();
+  };
 }
+
+

package/src/TUICallService/CallService/UIKitModal.ts

@@ -1,10 +1,17 @@
+
+// @if process.env.BUILD_TARGET ='MINI'
+import { UIKitModal } from '../UIKitModal';
+// @endif
+// @if process.env.BUILD_TARGET!='MINI'
 import TuiStore from '../TUIStore/tuiStore';
 import { StoreName } from '../const/call';
 import { NAME } from '../const/index';
-import { t } from '../locales'
+import { t } from '../locales';
 const TUIStore = TuiStore.getInstance();
+// @endif
 
-const MODAL_ERROR_CODES = [
+// @if process.env.BUILD_TARGET!='MINI'
+const WEB_MODAL_ERROR_CODES = [
     -1001,
     -1002,
     -1101,
@@ -14,9 +21,10 @@ const MODAL_ERROR_CODES = [
     -1201,
     30000,
     20007,
+    101002,
 ];
 
-const MODAL_ERROR_MAP = {
+const WEB_MODAL_ERROR_MAP = {
     '-1001': {
         id: 10001,
         key: 'error.10001'
@@ -52,19 +60,58 @@ const MODAL_ERROR_MAP = {
     '20007': {
         id: 10013,
         key: 'error.10013'
+    },
+    '101002': {
+        id: 10014,
+        key: 'error.10014'
     }
 };
+// @endif
+
+// @if process.env.BUILD_TARGET ='MINI'
+const MINI_MODAL_ERROR_CODES = [
+    -1001,
+    -1002,
+    101002,
+];
+
+const MINI_MODAL_ERROR_MAP = {
+    '-1001': {
+        id: 10001,
+        content: '您的应用还未开通音视频通话能力'
+    },
+    '-1002': {
+        id: 10002,
+        content: '您暂不支持使用该能力，请前往购买页购买开通'
+    },
+    '101002': {
+        id: 10014,
+        content: '发起通话失败，用户 ID 无效，请确认该用户已注册'
+    }
+}
+// @endif
 
 export function handleModalError(error) {
     if (!error || !error?.code) {
         return;
     }
 
-    if (!MODAL_ERROR_CODES.includes(error.code)) {
+    // @if process.env.BUILD_TARGET!='MINI'
+    handleWebModalError(error);
+    // @endif
+
+    // @if process.env.BUILD_TARGET='MINI'
+    handleMiniModalError(error);
+    // @endif
+}
+
+// @if process.env.BUILD_TARGET!='MINI'
+function handleWebModalError(error) {
+    if (!WEB_MODAL_ERROR_CODES.includes(error.code)) {
         return;
     }
 
-    const errorInfo = MODAL_ERROR_MAP[error.code.toString()];
+    const errorInfo = WEB_MODAL_ERROR_MAP[error.code.toString()];
     if (errorInfo) {
         let content = t(errorInfo.key);
         TUIStore.update(StoreName.CALL, NAME.MODAL_ERROR, {
@@ -73,4 +120,23 @@ export function handleModalError(error) {
             title: t('error')
         });
     }
-}
+}
+// @endif
+
+// @if process.env.BUILD_TARGET='MINI'
+function handleMiniModalError(error) {
+    if (!MINI_MODAL_ERROR_CODES.includes(error.code)) {
+        return;
+    }
+
+    const errorInfo = MINI_MODAL_ERROR_MAP[error.code.toString()];
+    if (errorInfo) {
+        UIKitModal.openModal({
+            id: errorInfo.id,
+            content: errorInfo.content,
+            title: '错误',
+            type: 'error'
+        });
+    }
+}
+// @endif

package/src/TUICallService/CallService/bellContext.ts

@@ -10,6 +10,7 @@ export class BellContext {
   private _calleeBellFilePath: string = DEFAULT_CALLEE_BELL_FILEPATH;
   private _callRole: string = CallRole.UNKNOWN;
   private _callStatus: string = CallStatus.IDLE;
+  private _isPlaying: boolean = false;
 
   constructor() {
     this._bellContext = new Audio();
@@ -39,14 +40,26 @@ export class BellContext {
   async play() {
     try {
       if (this._callStatus !== CallStatus.CALLING) {
-        return ;
+        return;
       }
+      // iOS 优化：增加更严格的状态检查
+      if (this._callStatus !== CallStatus.CALLING || this._isPlaying) {
+        console.warn(`${NAME.PREFIX}play skipped, callStatus: ${this._callStatus}, isPlaying: ${this._isPlaying}`);
+        return;
+      }
+      this._isPlaying = true;
       this.setBellSrc();
       if (this._callRole === CallRole.CALLEE && !this._isMuteBell) {
+        // 再次检查状态，避免在设置过程中状态已变更
+        if (!this._isPlaying || this._callStatus !== CallStatus.CALLING) return;
         await this._bellContext.play();
+        return;
       }
       if (this._callRole === CallRole.CALLER) {
+        // 再次检查状态，避免在设置过程中状态已变更
+        if (!this._isPlaying || this._callStatus !== CallStatus.CALLING) return;
         await this._bellContext.play();
+        return;
       }
     } catch (error) {
       console.warn(`${NAME.PREFIX}Failed to play audio file, ${error}`);
@@ -55,7 +68,8 @@ export class BellContext {
 
   async stop() {
     try {
-      await this._bellContext.pause();
+      this._isPlaying = false;
+      await this._bellContext?.pause();
     } catch (error) {
       console.warn(`${NAME.PREFIX}Failed to stop audio file, ${error}`);
     }
@@ -78,11 +92,20 @@ export class BellContext {
       this._calleeBellFilePath = '';
       this._callRole = CallRole.UNKNOWN;
       this._callStatus = CallStatus.IDLE;
+      this._isPlaying = false;
       this._bellContext.pause();
       this._bellContext = null;
     } catch (error) {
       console.warn(`${NAME.PREFIX}Failed to destroy, ${error}`);
     }
   }
+
+  private _delay(ms: number): any {
+    return new Promise((resolve: any) => setTimeout(resolve, ms));
+  }
+
+  private _delayCallback(callback: () => void, ms: number) {
+    setTimeout(callback, ms);
+  }
   
 }