@trtc/calls-uikit-react 4.2.4 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trtc/calls-uikit-react",
-  "version": "4.2.4",
+  "version": "4.4.0",
   "main": "./tuicall-uikit-react.umd.js",
   "module": "./tuicall-uikit-react.es.js",
   "types": "./types/index.d.ts",
@@ -14,8 +14,8 @@
   },
   "dependencies": {
     "@tencentcloud/tui-core-lite": "1.0.0",
-    "@trtc/call-engine-lite-js": "~3.4.7",
-    "@tencentcloud/lite-chat": "^1.5.0",
+    "@trtc/call-engine-lite-js": "~3.5.0",
+    "@tencentcloud/lite-chat": "^1.6.3",
     "@trtc/call-engine-lite-wx": "~3.4.7"
 },
   "bugs": {

package/src/Components/TUICallKit/TUICallKit.tsx

@@ -14,6 +14,7 @@ import {
 } from '../../TUICallService';
 import SingleCall from '../components/SingleCall/SingleCall';
 import GroupCall from '../components/GroupCall/GroupCall';
+import { AISubtitle } from '../components/common/AIAssistant';
 import { classNames } from '../util/classnames';
 import {
   CallInfoContext,
@@ -248,6 +249,7 @@ export default function TUICallKit(props: ITUICallKitProps) {
               <div style={{ ...style }} className={classnames} id='tuicallkit-id'>
                 {!isGroupCall && <SingleCall />}
                 {isGroupCall && <GroupCall />}
+                <AISubtitle />
               </div>
               <Portal attach="body">
                 <FloatingWindow

package/src/Components/components/common/AIAssistant/AIAssistant.ts

@@ -0,0 +1,132 @@
+import { mitt } from "./utils/index";
+
+interface IInitASRParams {
+  trtcCloudInstance: any;
+  getNickName?: (userId: string) => Promise<userInfo>;
+}
+
+interface ITranslationContent {
+  language: string;
+  content: string;
+}
+
+interface ITranslationInfo {
+  roundId: string;
+  sender: string;
+  nick?: string;
+  text: string;
+  end: boolean;
+  translation: ITranslationContent[];
+};
+
+export const ASREvent = {
+  TRANSCRIPTION: 'transcription',
+}
+
+const NAME = {
+  ON_RECV_CUSTOM_CMD_MESSAGE: 'onRecvCustomCmdMsg',
+  CUSTOM_MESSAGE_ASR_ID: 1,
+  USER_SPEAK: 10000,
+}
+
+interface userInfo {
+  userId: string;
+  nick: string;
+  avatar?: string;
+  [key: string]: any;
+}
+
+export default class AIAssistant {
+  emitter: any;
+  trtcCloudInstance: any;
+  getNickName: (userId: string) => Promise<userInfo>;
+  translationInfoList: ITranslationInfo[] = [];
+
+  constructor() {
+    this.emitter = mitt();
+    this.trtcCloudInstance = null;
+  };
+
+  initASR(params: IInitASRParams) {
+    const { trtcCloudInstance, getNickName } = params;
+    this.getNickName = getNickName;
+    this.trtcCloudInstance = trtcCloudInstance;
+    this.trtcCloudInstance.on(NAME.ON_RECV_CUSTOM_CMD_MESSAGE, this.handleCustomMessage, this);
+  };
+
+  on(type: string, handler: (event: any) => void) {
+    this.emitter.on(type, handler);
+  };
+
+  emit(type: string, events: any) {
+    this.emitter.emit(type, events)
+  };
+
+  off(type: string, handler: (event: any) => void) {
+    this.emitter.off(type, handler);
+  };
+
+  destroyASR() {
+    this.trtcCloudInstance.off(NAME.ON_RECV_CUSTOM_CMD_MESSAGE, this.handleCustomMessage, this);
+    this.translationInfoList = [];
+    this.trtcCloudInstance = null;
+  };
+
+  handleAIMessage = (data: any) => {
+    if (data?.type !== NAME.USER_SPEAK) return;
+    const { sender = '', payload } = data;
+    const { text = '', translation_text: translationText = '', end, roundid: roundId, translation_language: language = '' } = payload;
+
+    if (!roundId) return;
+    const translationInfo = this.translationInfoList.find(obj => obj.roundId === roundId);
+
+    if (translationInfo) {
+      translationInfo.text = text ? text : translationInfo.text;
+      translationInfo.end = end; // not change sender
+      const translationContent = (translationInfo.translation || []).find(obj => obj.language === language);
+      if (translationContent) {
+        translationContent.content = translationText;
+      } else if (language) {
+        translationInfo.translation.push({ language, content: translationText });
+      }
+    } else {
+      const translationContent: ITranslationContent = { language, content: translationText };
+      const translationValue: ITranslationInfo = {
+        roundId,
+        sender,
+        text,
+        end,
+        translation: language ? [translationContent] : [],
+      };
+      this.translationInfoList.push(translationValue);
+
+      // 异步获取昵称
+      this.getNickName?.(sender)?.then((res) => {
+        const item = this.translationInfoList.find(item => item.roundId === roundId);
+        if (item) {
+          item.nick = res?.nick || '';
+          this.emit(ASREvent.TRANSCRIPTION, { subtitleInfoList: this.translationInfoList });
+        }
+      });
+    }
+    this.emit(ASREvent.TRANSCRIPTION, { subtitleInfoList: this.translationInfoList });
+
+    if (end) {
+      const id = roundId;
+      setTimeout(() => {
+        if (this?.translationInfoList?.length > 0) {
+          this.translationInfoList = this.translationInfoList.filter(obj => obj.roundId !== id);
+          this.emit(ASREvent.TRANSCRIPTION, { subtitleInfoList: this.translationInfoList });
+        }
+      }, 8000);
+    }
+  };
+
+  handleCustomMessage = (userId: string, cmdId: number, seq: number, data: any) => {
+    if (cmdId === NAME.CUSTOM_MESSAGE_ASR_ID) {
+      const decodeData = new TextDecoder().decode(data);
+      const jsonData = JSON.parse(decodeData || '');
+      this.handleAIMessage(jsonData);
+    }
+  };
+}

package/src/Components/components/common/AIAssistant/AISubtitle.tsx

@@ -0,0 +1,89 @@
+import React, { useEffect, useState } from 'react';
+import { aiAssistant, ASREvent } from './index';
+
+interface TranslationContent {
+  language: string;
+  content: string;
+}
+
+interface SubtitleInfo {
+  roundId: string;
+  sender: string;
+  nick?: string;
+  text: string;
+  end: boolean;
+  translation: TranslationContent[];
+}
+
+interface AISubtitleProps {
+  customClass?: string;
+  customStyle?: React.CSSProperties;
+}
+
+const AISubtitle: React.FC<AISubtitleProps> = ({ customClass, customStyle }) => {
+  const [subtitleInfoList, setSubtitleInfoList] = useState<SubtitleInfo[]>([]);
+
+  const handleAISubtitle = (data: any) => {
+    if (!data?.subtitleInfoList) return;
+    setSubtitleInfoList(JSON.parse(JSON.stringify(data.subtitleInfoList)));
+  };
+
+  useEffect(() => {
+    aiAssistant.on(ASREvent.TRANSCRIPTION, handleAISubtitle);
+    
+    return () => {
+      aiAssistant.off(ASREvent.TRANSCRIPTION, handleAISubtitle);
+    };
+  }, []);
+
+  // 内联样式定义
+  const styles = {
+    subtitleContainer: {
+      position: 'absolute' as const,
+      zIndex: 100,
+      bottom: '120px',
+      left: '50%',
+      padding: '10px 12px',
+      color: '#fff',
+      backgroundColor: 'rgba(79, 88, 107, 0.7)',
+      borderRadius: '8px',
+      transform: 'translateX(-50%)',
+      width: '260px',
+      maxHeight: '280px',
+      overflowY: 'auto' as const,
+      overflowX: 'hidden' as const,
+    },
+    spacer: {
+      height: '16px',
+    },
+    senderName: {
+      color: 'yellow',
+    }
+  };
+
+  if (!subtitleInfoList.length) {
+    return null;
+  }
+
+  return (
+    <div
+      className={`ai-subtitle ${customClass || ''}`}
+      style={{ ...styles.subtitleContainer, ...customStyle }}
+    >
+      {subtitleInfoList.map((subtitleInfo, index) => (
+        <div key={subtitleInfo.roundId}>
+          {index !== 0 && <div style={{ height: '16px' }} />}
+          <div style={styles.senderName}>{`${subtitleInfo?.nick || subtitleInfo?.sender}:`}</div>
+          <div>{subtitleInfo?.text}</div>
+          {subtitleInfo?.translation?.map((translationContent, transIndex) => (
+            <div key={transIndex}>
+              <span>{`[${translationContent?.language}]: ${translationContent?.content}`}</span>
+            </div>
+          ))}
+        </div>
+      ))}
+    </div>
+  );
+};
+
+export default AISubtitle;

package/src/Components/components/common/AIAssistant/index.ts

@@ -0,0 +1,10 @@
+import AIAssistant, { ASREvent } from "./AIAssistant";
+import AISubtitle from "./AISubtitle";
+
+const aiAssistant = new AIAssistant();
+
+export {
+  aiAssistant,
+  ASREvent,
+  AISubtitle,
+}

package/src/Components/components/common/AIAssistant/utils/index.ts

@@ -0,0 +1,5 @@
+import mitt from './mitt';
+
+export {
+  mitt,
+}

package/src/Components/components/common/AIAssistant/utils/mitt.ts

@@ -0,0 +1,85 @@
+export type EventType = string | symbol;
+export type Handler<T = unknown> = (event: T) => void;
+export type WildcardHandler<T = Record<string, unknown>> = (
+	type: keyof T,
+	event: T[keyof T]
+) => void;
+export type EventHandlerList<T = unknown> = Array<Handler<T>>;
+export type WildCardEventHandlerList<T = Record<string, unknown>> = Array<
+	WildcardHandler<T>
+>;
+export type EventHandlerMap<Events extends Record<EventType, unknown>> = Map<
+	keyof Events | '*',
+	EventHandlerList<Events[keyof Events]> | WildCardEventHandlerList<Events>
+>;
+
+export interface Emitter<Events extends Record<EventType, unknown>> {
+	all: EventHandlerMap<Events>;
+
+	on<Key extends keyof Events>(type: Key, handler: Handler<Events[Key]>): void;
+	on(type: '*', handler: WildcardHandler<Events>): void;
+
+	off<Key extends keyof Events>(
+		type: Key,
+		handler?: Handler<Events[Key]>
+	): void;
+	off(type: '*', handler: WildcardHandler<Events>): void;
+
+	emit<Key extends keyof Events>(type: Key, event: Events[Key]): void;
+	emit<Key extends keyof Events>(
+		type: undefined extends Events[Key] ? Key : never
+	): void;
+}
+
+export default function mitt<Events extends Record<EventType, unknown>>(
+	all?: EventHandlerMap<Events>
+): Emitter<Events> {
+	type GenericEventHandler =
+		| Handler<Events[keyof Events]>
+		| WildcardHandler<Events>;
+	all = all || new Map();
+
+	return {
+		all,
+
+		on<Key extends keyof Events>(type: Key, handler: GenericEventHandler) {
+			const handlers: Array<GenericEventHandler> | undefined = all!.get(type);
+			if (handlers) {
+				handlers.push(handler);
+			} else {
+				all!.set(type, [handler] as EventHandlerList<Events[keyof Events]>);
+			}
+		},
+
+		off<Key extends keyof Events>(type: Key, handler?: GenericEventHandler) {
+			const handlers: Array<GenericEventHandler> | undefined = all!.get(type);
+			if (handlers) {
+				if (handler) {
+					handlers.splice(handlers.indexOf(handler) >>> 0, 1);
+				} else {
+					all!.set(type, []);
+				}
+			}
+		},
+
+		emit<Key extends keyof Events>(type: Key, evt?: Events[Key]) {
+			let handlers = all!.get(type);
+			if (handlers) {
+				(handlers as EventHandlerList<Events[keyof Events]>)
+					.slice()
+					.map((handler) => {
+						handler(evt!);
+					});
+			}
+
+			handlers = all!.get('*');
+			if (handlers) {
+				(handlers as WildCardEventHandlerList<Events>)
+					.slice()
+					.map((handler) => {
+						handler(type, evt!);
+					});
+			}
+		}
+	};
+}

package/src/TUICallService/CallService/AIAssistant.ts

@@ -0,0 +1,51 @@
+// @ts-ignore
+import { aiAssistant } from '../../Components/components/common/AIAssistant/index';
+import { getRemoteUserProfile } from './utils';
+
+interface IAIAssistant {
+  enableAISubtitle: (enable: boolean) => void;
+  setEngineInstance: (engineInstance: any) => void;
+  setImInstance: (imInstance: any) => void;
+};
+
+export class AIAssistant implements IAIAssistant {
+  static instance: IAIAssistant;
+  private _tuiCallEngine: any = null;
+  private _imInstance: any = null;
+
+  static getInstance() {
+    if (!AIAssistant.instance) {
+      AIAssistant.instance = new AIAssistant();
+    }
+    return AIAssistant.instance;
+  }
+
+  public enableAISubtitle(enable: boolean): void {
+    this._tuiCallEngine?.reportLog?.({
+      name: 'TUICallKit.enableAISubtitle.start',
+      data: { enable },
+    });
+
+    const trtcCloudInstance = this._tuiCallEngine.getTRTCCloudInstance();
+    const getNickName = async(userId: string) => {
+      const res = await getRemoteUserProfile([userId], this._imInstance);
+      return res.length ? res[0] : [];
+    }
+    if (enable) {
+      aiAssistant.initASR({
+        trtcCloudInstance,
+        // @ts-ignore
+        getNickName,
+      });
+    } else {
+      aiAssistant.destroyASR();
+    }
+  }
+
+  public setEngineInstance(engineInstance: any): void {
+    this._tuiCallEngine = engineInstance;
+  }
+  public setImInstance(imInstance: any): void {
+    this._imInstance = imInstance;
+  }
+}

package/src/TUICallService/CallService/index.ts

@@ -21,12 +21,14 @@ import TuiGlobal from '../TUIGlobal/tuiGlobal';
 import TuiStore from '../TUIStore/tuiStore';
 import { UIDesign } from './UIDesign';
 import ChatCombine from './chatCombine';
+import { AIAssistant } from './AIAssistant';
 import EngineEventHandler from './engineEventHandler';
 const TUIGlobal: ITUIGlobal = TuiGlobal.getInstance();
 const TUIStore: ITUIStore = TuiStore.getInstance();
 const uiDesign = UIDesign.getInstance();
 uiDesign.setTUIStore(TUIStore);
-const version = '4.2.4';
+const aiAssistant = AIAssistant.getInstance();
+const version = '4.4.0';
 const frameWork = 'react';
 export { TUIGlobal, TUIStore, uiDesign };
 
@@ -116,8 +118,18 @@ export default class TUICallService {
       TUIStore.update(StoreName.CALL, NAME.LOCAL_USER_INFO, { userId: userID });
       TUIStore.update(StoreName.CALL, NAME.LOCAL_USER_INFO_EXCLUDE_VOLUMN, { userId: userID });
       uiDesign.updateViewBackgroundUserId('local');
+      
+      aiAssistant.setEngineInstance(this._tuiCallEngine);
+      aiAssistant.setImInstance(this.getTim());
+
+      this.enableAISubtitle(true);
       await this._tuiCallEngine.login({ userID, userSig, assetsPath: '' }); // web && mini
       const uiConfig = TUIStore.getData(StoreName.CALL, NAME.CUSTOM_UI_CONFIG);
+      // close auto play dialog
+      const trtcCloudInstance = this._tuiCallEngine?.getTRTCCloudInstance?.()
+      trtcCloudInstance?.callExperimentalAPI(JSON.stringify(
+        { "api": "enableAutoPlayDialog","params": { "enable": 0 } }
+      ));
       this._tuiCallEngine?.reportLog?.({
         name: 'TUICallkit.init',
         data: {
@@ -291,6 +303,9 @@ export default class TUICallService {
   public setCameraDefaultState(isOpen: boolean) {
     uiDesign.setCameraDefaultState(isOpen);
   }
+  public enableAISubtitle(enable: boolean): void {
+    aiAssistant.enableAISubtitle(enable);
+  }
   // =============================【实验性接口】=============================
   public callExperimentalAPI(jsonStr: string) {
     const jsonObj = JSON.parse(jsonStr);

package/src/TUICallService/locales/en.ts

@@ -66,6 +66,8 @@ export const en = {
   "The network is poor during your current call": "The network is poor during your current call",
   "The other user network is poor during the current call": "The other party's network is poor during the current call",
   "TUICallKit init is not complete": "TUICallKit init is not complete. You need to use this API after the init API is finished.",
+  "auto play failed": "Audio playback was blocked",
+  "click to resume": "Resume playback",
   // combine chat
   "Video call": "Video call",
   "Voice call": "Voice call",

package/src/TUICallService/locales/ja_JP.ts

@@ -77,6 +77,8 @@ export const ja_JP = {
   "The network is poor during your current call": "現在の通話で、あなたのネットワークは不良です",
   "The other user network is poor during the current call": "現在の通話で、相手側のネットワークが不良です",
   "TUICallKit init is not complete": "TUICallKitの初期化ログインが完了していません。init が完了した後にこのAPIを使用する必要があります。",
+  "auto play failed": "オーディオ再生がブロックされました",
+  "click to resume": "再生を再開",
   // 待废弃文案
   'timeout': '{{ userList }} タイムアウト',
   'kick out': 'キックアウトされました',

package/src/TUICallService/locales/zh-cn.ts

@@ -66,6 +66,8 @@ export const zh = {
   "The network is poor during your current call": "当前通话你的网络不佳",
   "The other user network is poor during the current call": "当前通话对方网络不佳",
   "TUICallKit init is not complete": "TUICallKit 初始化登录未完成，需要在 init 完成后使用此 API",
+  "auto play failed": "音频播放被拦截，请点击恢复播放",
+  "click to resume": "恢复播放",
   // combine chat
   "Video call": "发起视频通话",
   "Voice call": "发起语音通话",

package/src/TUICallService/utils/common-utils.ts

@@ -143,7 +143,7 @@ export const retryPromise = (promise: Promise<any>, num: number = 6, time: numbe
  * @returns {Boolean}
  */
 export function handleRepeatedCallError(error: any) {
-  if (error?.message.indexOf('is ongoing, please avoid repeated calls') !== -1) {
+  if (error?.message?.indexOf('is ongoing, please avoid repeated calls') !== -1) {
     return true;
   }
   return false;

package/src/index.ts

@@ -35,7 +35,7 @@ const TUICallType = {
   AUDIO_CALL: 1,
   VIDEO_CALL: 2,
 };
-const Version = '4.2.4'; // basic-demo 原来上报使用
+const Version = '4.4.0'; // basic-demo 原来上报使用
 
 // 输出产物
 export {