@coze/realtime-api 1.2.0 → 1.2.1-beta.11

package/dist/umd/index.js

@@ -96,6 +96,7 @@
             checkDevicePermission: ()=>checkDevicePermission,
             checkPermission: ()=>checkPermission,
             getAudioDevices: ()=>getAudioDevices,
+            isMobileVideoDevice: ()=>isMobileVideoDevice,
             isScreenShareDevice: ()=>isScreenShareDevice,
             isScreenShareSupported: ()=>isScreenShareSupported,
             sleep: ()=>sleep
@@ -173,6 +174,11 @@
         };
         const isScreenShareDevice = (deviceId)=>'screenShare' === deviceId;
         /**
+ * 判断是否前后置摄像头
+ * @param deviceId
+ * @returns
+ */ const isMobileVideoDevice = (deviceId)=>'user' === deviceId || 'environment' === deviceId;
+        /**
  * Check if browser supports screen sharing
  * 检查浏览器是否支持屏幕共享
  */ function isScreenShareSupported() {
@@ -193,6 +199,10 @@
    * zh: 所有服务端事件
    */ EventNames["ALL_SERVER"] = "server.*";
             /**
+   * en: Room info
+   * zh: 房间信息
+   */ EventNames["ROOM_INFO"] = "client.room.info";
+            /**
    * en: Client connected
    * zh: 客户端连接
    */ EventNames["CONNECTED"] = "client.connected";
@@ -225,6 +235,10 @@
    * zh: 客户端视频关闭
    */ EventNames["VIDEO_OFF"] = "client.video.off";
             /**
+   * en: Client video error
+   * zh: 客户端视频(或屏幕共享）错误
+   */ EventNames["VIDEO_ERROR"] = "client.video.error";
+            /**
    * en: Client video event
    * zh: 客户端视频事件
    */ EventNames["PLAYER_EVENT"] = "client.video.event";
@@ -291,7 +305,51 @@
             /**
    * en: Session updated
    * zh: 会话更新
-   */ EventNames["SESSION_UPDATE"] = "server.session.update";
+   */ EventNames["SESSION_UPDATED"] = "server.session.updated";
+            /**
+   * en: Conversation created
+   * zh: 会话创建
+   */ EventNames["CONVERSATION_CREATED"] = "server.conversation.created";
+            /**
+   * en: Conversation chat created
+   * zh: 会话对话创建
+   */ EventNames["CONVERSATION_CHAT_CREATED"] = "server.conversation.chat.created";
+            /**
+   * en: Conversation chat in progress
+   * zh: 对话正在处理中
+   */ EventNames["CONVERSATION_CHAT_IN_PROGRESS"] = "server.conversation.chat.in_progress";
+            /**
+   * en: Conversation message delta received
+   * zh: 文本消息增量返回
+   */ EventNames["CONVERSATION_MESSAGE_DELTA"] = "server.conversation.message.delta";
+            /**
+   * en: Conversation message completed
+   * zh: 文本消息完成
+   */ EventNames["CONVERSATION_MESSAGE_COMPLETED"] = "server.conversation.message.completed";
+            /**
+   * en: Conversation chat completed
+   * zh: 对话完成
+   */ EventNames["CONVERSATION_CHAT_COMPLETED"] = "server.conversation.chat.completed";
+            /**
+   * en: Conversation chat requires action
+   * zh: 对话需要插件
+   */ EventNames["CONVERSATION_CHAT_REQUIRES_ACTION"] = "server.conversation.chat.requires_action";
+            /**
+   * en: Conversation chat failed
+   * zh: 对话失败
+   */ EventNames["CONVERSATION_CHAT_FAILED"] = "server.conversation.chat.failed";
+            /**
+   * en: Session pre answer updated
+   * zh: 安抚配置更新成功
+   */ EventNames["SESSION_PRE_ANSWER_UPDATED"] = "server.session.pre_answer.updated";
+            /**
+   * en: Conversation audio transcript delta
+   * zh: 用户语音识别字幕
+   */ EventNames["CONVERSATION_AUDIO_TRANSCRIPT_DELTA"] = "server.conversation.audio_transcript.delta";
+            /**
+   * en: Mode updated
+   * zh: 更新房间模式成功
+   */ EventNames["MODE_UPDATED"] = "server.mode.updated";
             return EventNames;
         }(event_names_EventNames || {});
         /* ESM default export */ const event_names = event_names_EventNames;
@@ -382,6 +440,7 @@
                 this.engine.on(rtc_default().events.onUserLeave, this.handleUserLeave);
                 this.engine.on(rtc_default().events.onError, this.handleEventError);
                 this.engine.on(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
+                this.engine.on(rtc_default().events.onTrackEnded, this.handleTrackEnded);
                 if (this._isSupportVideo) this.engine.on(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
                 if (this._debug) {
                     this.engine.on(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
@@ -394,6 +453,7 @@
                 this.engine.off(rtc_default().events.onUserLeave, this.handleUserLeave);
                 this.engine.off(rtc_default().events.onError, this.handleEventError);
                 this.engine.off(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
+                this.engine.off(rtc_default().events.onTrackEnded, this.handleTrackEnded);
                 if (this._isSupportVideo) this.engine.off(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
                 if (this._debug) {
                     this.engine.off(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
@@ -444,6 +504,10 @@
                     downlinkNetworkQuality
                 });
             }
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            handleTrackEnded(event) {
+                if ((null == event ? void 0 : event.kind) === 'video') this.dispatch(event_names.VIDEO_OFF, event);
+            }
             async joinRoom(options) {
                 const { token, roomId, uid, audioMutedDefault, videoOnDefault, isAutoSubscribeAudio } = options;
                 try {
@@ -477,7 +541,11 @@
                 const devices = await getAudioDevices({
                     video: true
                 });
-                if (-1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
+                if (!isMobileVideoDevice(deviceId) && -1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
+                this.engine.setLocalVideoPlayer(isScreenShareDevice(deviceId) ? rtc_.StreamIndex.STREAM_INDEX_SCREEN : rtc_.StreamIndex.STREAM_INDEX_MAIN, {
+                    renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
+                    userId: this._roomUserId
+                });
                 await this.changeVideoState(false);
                 if (isScreenShareDevice(deviceId)) {
                     if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) this.engine.setLocalVideoPlayer(rtc_.StreamIndex.STREAM_INDEX_MAIN);
@@ -493,10 +561,6 @@
                     if (isAutoCapture) await this.engine.startVideoCapture(deviceId);
                     this._streamIndex = rtc_.StreamIndex.STREAM_INDEX_MAIN;
                 }
-                this.engine.setLocalVideoPlayer(this._streamIndex, {
-                    renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
-                    userId: this._roomUserId
-                });
             }
             async createLocalStream(userId, videoConfig) {
                 this._roomUserId = userId;
@@ -506,7 +570,12 @@
                 if (!devices.audioInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get audio devices');
                 if (this._isSupportVideo && !devices.videoInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get video devices');
                 await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
-                if (this._isSupportVideo) this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
+                if (this._isSupportVideo) try {
+                    await this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
+                    this.dispatch((null == videoConfig ? void 0 : videoConfig.videoOnDefault) ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
+                } catch (e) {
+                    this.dispatch(event_names.VIDEO_ERROR, e);
+                }
             }
             async disconnect() {
                 try {
@@ -529,23 +598,18 @@
                 }
             }
             async changeVideoState(isVideoOn) {
-                try {
-                    if (isVideoOn) {
-                        if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
-                        else {
-                            var _this__videoConfig;
-                            this.engine.setVideoSourceType(rtc_.StreamIndex.STREAM_INDEX_SCREEN, rtc_.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
-                            await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
-                            await this.engine.publishScreen(rtc_.MediaType.VIDEO);
-                        }
-                    } else if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
+                if (isVideoOn) {
+                    if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
                     else {
-                        await this.engine.stopScreenCapture();
-                        await this.engine.unpublishScreen(rtc_.MediaType.VIDEO);
+                        var _this__videoConfig;
+                        this.engine.setVideoSourceType(rtc_.StreamIndex.STREAM_INDEX_SCREEN, rtc_.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
+                        await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
+                        await this.engine.publishScreen(rtc_.MediaType.VIDEO);
                     }
-                } catch (e) {
-                    this.dispatch(event_names.ERROR, e);
-                    throw e;
+                } else if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
+                else {
+                    await this.engine.stopScreenCapture();
+                    await this.engine.unpublishScreen(rtc_.MediaType.VIDEO);
                 }
             }
             async stop() {
@@ -635,6 +699,7 @@
                 this.handleEventError = this.handleEventError.bind(this);
                 this.handlePlayerEvent = this.handlePlayerEvent.bind(this);
                 this.handleNetworkQuality = this.handleNetworkQuality.bind(this);
+                this.handleTrackEnded = this.handleTrackEnded.bind(this);
                 // Debug only
                 this.handleLocalAudioPropertiesReport = this.handleLocalAudioPropertiesReport.bind(this);
                 this.handleRemoteAudioPropertiesReport = this.handleRemoteAudioPropertiesReport.bind(this);
@@ -658,16 +723,17 @@
                     // Step1 get token
                     if (getRoomInfo) roomInfo = await getRoomInfo();
                     else {
-                        let config;
-                        if (this._config.videoConfig) config = isScreenShareDevice(this._config.videoConfig.videoInputDeviceId) ? {
-                            video_config: {
+                        const config = {};
+                        if (this._config.prologueContent) config.prologue_content = this._config.prologueContent;
+                        if (void 0 !== this._config.roomMode && null !== this._config.roomMode) config.room_mode = this._config.roomMode;
+                        if (this._config.videoConfig) {
+                            if (isScreenShareDevice(this._config.videoConfig.videoInputDeviceId)) config.video_config = {
                                 stream_video_type: 'screen'
-                            }
-                        } : {
-                            video_config: {
+                            };
+                            else config.video_config = {
                                 stream_video_type: 'main'
-                            }
-                        };
+                            };
+                        }
                         roomInfo = await this._api.audio.rooms.create({
                             bot_id: botId,
                             conversation_id: conversationId || void 0,
@@ -682,6 +748,12 @@
                     this.dispatch(event_names.ERROR, error);
                     throw new RealtimeAPIError(error_RealtimeError.CREATE_ROOM_ERROR, error instanceof Error ? error.message : 'Unknown error', error);
                 }
+                this.dispatch(event_names.ROOM_INFO, {
+                    roomId: roomInfo.room_id,
+                    uid: roomInfo.uid,
+                    token: roomInfo.token,
+                    appId: roomInfo.app_id
+                });
                 this._isTestEnv = TEST_APP_ID === roomInfo.app_id;
                 // Step2 create engine
                 this._client = new EngineClient(roomInfo.app_id, this._config.debug, this._isTestEnv, this._isSupportVideo, this._config.videoConfig);
@@ -763,10 +835,14 @@
                 else this.dispatch(event_names.AUDIO_MUTED, {});
             }
             async setVideoEnable(isEnable) {
-                var _this__client;
-                await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
-                if (isEnable) this.dispatch(event_names.VIDEO_ON, {});
-                else this.dispatch(event_names.VIDEO_OFF, {});
+                try {
+                    var _this__client;
+                    await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
+                    this.dispatch(isEnable ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
+                } catch (e) {
+                    this.dispatch(event_names.VIDEO_ERROR, e);
+                    throw e;
+                }
             }
             /**
    * en: Enable audio properties reporting (debug mode only)
@@ -823,9 +899,19 @@
                     deviceId
                 });
             }
-            async setVideoInputDevice(deviceId) {
-                var _this__client;
-                await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
+            /**
+   * en: Set the video input device
+   *
+   * zh: 设置视频输入设备
+   */ async setVideoInputDevice(deviceId) {
+                try {
+                    var _this__client;
+                    await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
+                    this.dispatch(event_names.VIDEO_ON, {});
+                } catch (e) {
+                    this.dispatch(event_names.VIDEO_ERROR, e);
+                    throw e;
+                }
                 this.dispatch(event_names.VIDEO_INPUT_DEVICE_CHANGED, {
                     deviceId
                 });
@@ -880,6 +966,8 @@
    *                                               可选，视频输入设备的设备 ID。
    * @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
    *                                         可选，屏幕共享配置，如果 videoInputDeviceId 是 'screenShare'，请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
+   * @param config.prologueContent - Optional, Prologue content. | 可选，开场白内容。
+   * @param config.roomMode - Optional, Room mode. | 可选，房间模式。
    */ constructor(config){
                 super(config.debug), this._client = null, this.isConnected = false, this._isTestEnv = false, this._isSupportVideo = false;
                 this._config = config;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coze/realtime-api",
-  "version": "1.2.0",
+  "version": "1.2.1-beta.11",
   "description": "A powerful real-time communication SDK for voice interactions with Coze AI bots | 扣子官方实时通信 SDK，用于与 Coze AI bots 进行语音交互",
   "keywords": [
     "coze",
@@ -54,8 +54,8 @@
     "test:cov": "vitest --coverage --run"
   },
   "dependencies": {
-    "@coze/api": "1.2.0",
-    "@volcengine/rtc": "^4.62.1"
+    "@coze/api": "1.2.1-beta.23",
+    "@volcengine/rtc": "~4.62.11"
   },
   "devDependencies": {
     "@coze-infra/eslint-config": "workspace:*",
@@ -66,10 +66,10 @@
     "@types/node": "^20",
     "@types/uuid": "^9.0.1",
     "@types/whatwg-fetch": "^0.0.33",
-    "@vitest/coverage-v8": "~2.1.4",
+    "@vitest/coverage-v8": "~2.1.9",
     "axios": "^1.7.7",
     "typescript": "^5.5.3",
-    "vitest": "~2.1.4"
+    "vitest": "~2.1.9"
   },
   "cozePublishConfig": {
     "exports": {