@coze/realtime-api 1.2.0 → 1.2.1-beta.11

package/dist/cjs/event-names/index.js

@@ -49,6 +49,10 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 所有服务端事件
    */ EventNames["ALL_SERVER"] = "server.*";
     /**
+   * en: Room info
+   * zh: 房间信息
+   */ EventNames["ROOM_INFO"] = "client.room.info";
+    /**
    * en: Client connected
    * zh: 客户端连接
    */ EventNames["CONNECTED"] = "client.connected";
@@ -81,6 +85,10 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 客户端视频关闭
    */ EventNames["VIDEO_OFF"] = "client.video.off";
     /**
+   * en: Client video error
+   * zh: 客户端视频(或屏幕共享）错误
+   */ EventNames["VIDEO_ERROR"] = "client.video.error";
+    /**
    * en: Client video event
    * zh: 客户端视频事件
    */ EventNames["PLAYER_EVENT"] = "client.video.event";
@@ -147,7 +155,51 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
     /**
    * en: Session updated
    * zh: 会话更新
-   */ EventNames["SESSION_UPDATE"] = "server.session.update";
+   */ EventNames["SESSION_UPDATED"] = "server.session.updated";
+    /**
+   * en: Conversation created
+   * zh: 会话创建
+   */ EventNames["CONVERSATION_CREATED"] = "server.conversation.created";
+    /**
+   * en: Conversation chat created
+   * zh: 会话对话创建
+   */ EventNames["CONVERSATION_CHAT_CREATED"] = "server.conversation.chat.created";
+    /**
+   * en: Conversation chat in progress
+   * zh: 对话正在处理中
+   */ EventNames["CONVERSATION_CHAT_IN_PROGRESS"] = "server.conversation.chat.in_progress";
+    /**
+   * en: Conversation message delta received
+   * zh: 文本消息增量返回
+   */ EventNames["CONVERSATION_MESSAGE_DELTA"] = "server.conversation.message.delta";
+    /**
+   * en: Conversation message completed
+   * zh: 文本消息完成
+   */ EventNames["CONVERSATION_MESSAGE_COMPLETED"] = "server.conversation.message.completed";
+    /**
+   * en: Conversation chat completed
+   * zh: 对话完成
+   */ EventNames["CONVERSATION_CHAT_COMPLETED"] = "server.conversation.chat.completed";
+    /**
+   * en: Conversation chat requires action
+   * zh: 对话需要插件
+   */ EventNames["CONVERSATION_CHAT_REQUIRES_ACTION"] = "server.conversation.chat.requires_action";
+    /**
+   * en: Conversation chat failed
+   * zh: 对话失败
+   */ EventNames["CONVERSATION_CHAT_FAILED"] = "server.conversation.chat.failed";
+    /**
+   * en: Session pre answer updated
+   * zh: 安抚配置更新成功
+   */ EventNames["SESSION_PRE_ANSWER_UPDATED"] = "server.session.pre_answer.updated";
+    /**
+   * en: Conversation audio transcript delta
+   * zh: 用户语音识别字幕
+   */ EventNames["CONVERSATION_AUDIO_TRANSCRIPT_DELTA"] = "server.conversation.audio_transcript.delta";
+    /**
+   * en: Mode updated
+   * zh: 更新房间模式成功
+   */ EventNames["MODE_UPDATED"] = "server.mode.updated";
     return EventNames;
 }(EventNames || {});
 /* ESM default export */ const __WEBPACK_DEFAULT_EXPORT__ = EventNames;

package/dist/cjs/index.js

@@ -61,6 +61,7 @@ __webpack_require__.d(utils_namespaceObject, {
     checkDevicePermission: ()=>checkDevicePermission,
     checkPermission: ()=>checkPermission,
     getAudioDevices: ()=>getAudioDevices,
+    isMobileVideoDevice: ()=>isMobileVideoDevice,
     isScreenShareDevice: ()=>isScreenShareDevice,
     isScreenShareSupported: ()=>isScreenShareSupported,
     sleep: ()=>sleep
@@ -135,6 +136,11 @@ var rtc_default = /*#__PURE__*/ __webpack_require__.n(rtc_namespaceObject);
     };
 };
 const isScreenShareDevice = (deviceId)=>'screenShare' === deviceId;
+/**
+ * 判断是否前后置摄像头
+ * @param deviceId
+ * @returns
+ */ const isMobileVideoDevice = (deviceId)=>'user' === deviceId || 'environment' === deviceId;
 /**
  * Check if browser supports screen sharing
  * 检查浏览器是否支持屏幕共享
@@ -156,6 +162,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 所有服务端事件
    */ EventNames["ALL_SERVER"] = "server.*";
     /**
+   * en: Room info
+   * zh: 房间信息
+   */ EventNames["ROOM_INFO"] = "client.room.info";
+    /**
    * en: Client connected
    * zh: 客户端连接
    */ EventNames["CONNECTED"] = "client.connected";
@@ -188,6 +198,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 客户端视频关闭
    */ EventNames["VIDEO_OFF"] = "client.video.off";
     /**
+   * en: Client video error
+   * zh: 客户端视频(或屏幕共享）错误
+   */ EventNames["VIDEO_ERROR"] = "client.video.error";
+    /**
    * en: Client video event
    * zh: 客户端视频事件
    */ EventNames["PLAYER_EVENT"] = "client.video.event";
@@ -254,7 +268,51 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
     /**
    * en: Session updated
    * zh: 会话更新
-   */ EventNames["SESSION_UPDATE"] = "server.session.update";
+   */ EventNames["SESSION_UPDATED"] = "server.session.updated";
+    /**
+   * en: Conversation created
+   * zh: 会话创建
+   */ EventNames["CONVERSATION_CREATED"] = "server.conversation.created";
+    /**
+   * en: Conversation chat created
+   * zh: 会话对话创建
+   */ EventNames["CONVERSATION_CHAT_CREATED"] = "server.conversation.chat.created";
+    /**
+   * en: Conversation chat in progress
+   * zh: 对话正在处理中
+   */ EventNames["CONVERSATION_CHAT_IN_PROGRESS"] = "server.conversation.chat.in_progress";
+    /**
+   * en: Conversation message delta received
+   * zh: 文本消息增量返回
+   */ EventNames["CONVERSATION_MESSAGE_DELTA"] = "server.conversation.message.delta";
+    /**
+   * en: Conversation message completed
+   * zh: 文本消息完成
+   */ EventNames["CONVERSATION_MESSAGE_COMPLETED"] = "server.conversation.message.completed";
+    /**
+   * en: Conversation chat completed
+   * zh: 对话完成
+   */ EventNames["CONVERSATION_CHAT_COMPLETED"] = "server.conversation.chat.completed";
+    /**
+   * en: Conversation chat requires action
+   * zh: 对话需要插件
+   */ EventNames["CONVERSATION_CHAT_REQUIRES_ACTION"] = "server.conversation.chat.requires_action";
+    /**
+   * en: Conversation chat failed
+   * zh: 对话失败
+   */ EventNames["CONVERSATION_CHAT_FAILED"] = "server.conversation.chat.failed";
+    /**
+   * en: Session pre answer updated
+   * zh: 安抚配置更新成功
+   */ EventNames["SESSION_PRE_ANSWER_UPDATED"] = "server.session.pre_answer.updated";
+    /**
+   * en: Conversation audio transcript delta
+   * zh: 用户语音识别字幕
+   */ EventNames["CONVERSATION_AUDIO_TRANSCRIPT_DELTA"] = "server.conversation.audio_transcript.delta";
+    /**
+   * en: Mode updated
+   * zh: 更新房间模式成功
+   */ EventNames["MODE_UPDATED"] = "server.mode.updated";
     return EventNames;
 }(event_names_EventNames || {});
 /* ESM default export */ const event_names = event_names_EventNames;
@@ -344,6 +402,7 @@ class EngineClient extends RealtimeEventHandler {
         this.engine.on(rtc_default().events.onUserLeave, this.handleUserLeave);
         this.engine.on(rtc_default().events.onError, this.handleEventError);
         this.engine.on(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
+        this.engine.on(rtc_default().events.onTrackEnded, this.handleTrackEnded);
         if (this._isSupportVideo) this.engine.on(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
         if (this._debug) {
             this.engine.on(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
@@ -356,6 +415,7 @@ class EngineClient extends RealtimeEventHandler {
         this.engine.off(rtc_default().events.onUserLeave, this.handleUserLeave);
         this.engine.off(rtc_default().events.onError, this.handleEventError);
         this.engine.off(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
+        this.engine.off(rtc_default().events.onTrackEnded, this.handleTrackEnded);
         if (this._isSupportVideo) this.engine.off(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
         if (this._debug) {
             this.engine.off(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
@@ -406,6 +466,10 @@ class EngineClient extends RealtimeEventHandler {
             downlinkNetworkQuality
         });
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    handleTrackEnded(event) {
+        if ((null == event ? void 0 : event.kind) === 'video') this.dispatch(event_names.VIDEO_OFF, event);
+    }
     async joinRoom(options) {
         const { token, roomId, uid, audioMutedDefault, videoOnDefault, isAutoSubscribeAudio } = options;
         try {
@@ -439,7 +503,11 @@ class EngineClient extends RealtimeEventHandler {
         const devices = await getAudioDevices({
             video: true
         });
-        if (-1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
+        if (!isMobileVideoDevice(deviceId) && -1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
+        this.engine.setLocalVideoPlayer(isScreenShareDevice(deviceId) ? rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN : rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN, {
+            renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
+            userId: this._roomUserId
+        });
         await this.changeVideoState(false);
         if (isScreenShareDevice(deviceId)) {
             if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) this.engine.setLocalVideoPlayer(rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN);
@@ -455,10 +523,6 @@ class EngineClient extends RealtimeEventHandler {
             if (isAutoCapture) await this.engine.startVideoCapture(deviceId);
             this._streamIndex = rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN;
         }
-        this.engine.setLocalVideoPlayer(this._streamIndex, {
-            renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
-            userId: this._roomUserId
-        });
     }
     async createLocalStream(userId, videoConfig) {
         this._roomUserId = userId;
@@ -468,7 +532,12 @@ class EngineClient extends RealtimeEventHandler {
         if (!devices.audioInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get audio devices');
         if (this._isSupportVideo && !devices.videoInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get video devices');
         await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
-        if (this._isSupportVideo) this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
+        if (this._isSupportVideo) try {
+            await this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
+            this.dispatch((null == videoConfig ? void 0 : videoConfig.videoOnDefault) ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
+        } catch (e) {
+            this.dispatch(event_names.VIDEO_ERROR, e);
+        }
     }
     async disconnect() {
         try {
@@ -491,23 +560,18 @@ class EngineClient extends RealtimeEventHandler {
         }
     }
     async changeVideoState(isVideoOn) {
-        try {
-            if (isVideoOn) {
-                if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
-                else {
-                    var _this__videoConfig;
-                    this.engine.setVideoSourceType(rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN, rtc_namespaceObject.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
-                    await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
-                    await this.engine.publishScreen(rtc_namespaceObject.MediaType.VIDEO);
-                }
-            } else if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
+        if (isVideoOn) {
+            if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
             else {
-                await this.engine.stopScreenCapture();
-                await this.engine.unpublishScreen(rtc_namespaceObject.MediaType.VIDEO);
+                var _this__videoConfig;
+                this.engine.setVideoSourceType(rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN, rtc_namespaceObject.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
+                await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
+                await this.engine.publishScreen(rtc_namespaceObject.MediaType.VIDEO);
             }
-        } catch (e) {
-            this.dispatch(event_names.ERROR, e);
-            throw e;
+        } else if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
+        else {
+            await this.engine.stopScreenCapture();
+            await this.engine.unpublishScreen(rtc_namespaceObject.MediaType.VIDEO);
         }
     }
     async stop() {
@@ -597,6 +661,7 @@ class EngineClient extends RealtimeEventHandler {
         this.handleEventError = this.handleEventError.bind(this);
         this.handlePlayerEvent = this.handlePlayerEvent.bind(this);
         this.handleNetworkQuality = this.handleNetworkQuality.bind(this);
+        this.handleTrackEnded = this.handleTrackEnded.bind(this);
         // Debug only
         this.handleLocalAudioPropertiesReport = this.handleLocalAudioPropertiesReport.bind(this);
         this.handleRemoteAudioPropertiesReport = this.handleRemoteAudioPropertiesReport.bind(this);
@@ -620,16 +685,17 @@ class RealtimeClient extends RealtimeEventHandler {
             // Step1 get token
             if (getRoomInfo) roomInfo = await getRoomInfo();
             else {
-                let config;
-                if (this._config.videoConfig) config = isScreenShareDevice(this._config.videoConfig.videoInputDeviceId) ? {
-                    video_config: {
+                const config = {};
+                if (this._config.prologueContent) config.prologue_content = this._config.prologueContent;
+                if (void 0 !== this._config.roomMode && null !== this._config.roomMode) config.room_mode = this._config.roomMode;
+                if (this._config.videoConfig) {
+                    if (isScreenShareDevice(this._config.videoConfig.videoInputDeviceId)) config.video_config = {
                         stream_video_type: 'screen'
-                    }
-                } : {
-                    video_config: {
+                    };
+                    else config.video_config = {
                         stream_video_type: 'main'
-                    }
-                };
+                    };
+                }
                 roomInfo = await this._api.audio.rooms.create({
                     bot_id: botId,
                     conversation_id: conversationId || void 0,
@@ -644,6 +710,12 @@ class RealtimeClient extends RealtimeEventHandler {
             this.dispatch(event_names.ERROR, error);
             throw new RealtimeAPIError(error_RealtimeError.CREATE_ROOM_ERROR, error instanceof Error ? error.message : 'Unknown error', error);
         }
+        this.dispatch(event_names.ROOM_INFO, {
+            roomId: roomInfo.room_id,
+            uid: roomInfo.uid,
+            token: roomInfo.token,
+            appId: roomInfo.app_id
+        });
         this._isTestEnv = TEST_APP_ID === roomInfo.app_id;
         // Step2 create engine
         this._client = new EngineClient(roomInfo.app_id, this._config.debug, this._isTestEnv, this._isSupportVideo, this._config.videoConfig);
@@ -725,10 +797,14 @@ class RealtimeClient extends RealtimeEventHandler {
         else this.dispatch(event_names.AUDIO_MUTED, {});
     }
     async setVideoEnable(isEnable) {
-        var _this__client;
-        await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
-        if (isEnable) this.dispatch(event_names.VIDEO_ON, {});
-        else this.dispatch(event_names.VIDEO_OFF, {});
+        try {
+            var _this__client;
+            await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
+            this.dispatch(isEnable ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
+        } catch (e) {
+            this.dispatch(event_names.VIDEO_ERROR, e);
+            throw e;
+        }
     }
     /**
    * en: Enable audio properties reporting (debug mode only)
@@ -785,9 +861,19 @@ class RealtimeClient extends RealtimeEventHandler {
             deviceId
         });
     }
-    async setVideoInputDevice(deviceId) {
-        var _this__client;
-        await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
+    /**
+   * en: Set the video input device
+   *
+   * zh: 设置视频输入设备
+   */ async setVideoInputDevice(deviceId) {
+        try {
+            var _this__client;
+            await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
+            this.dispatch(event_names.VIDEO_ON, {});
+        } catch (e) {
+            this.dispatch(event_names.VIDEO_ERROR, e);
+            throw e;
+        }
         this.dispatch(event_names.VIDEO_INPUT_DEVICE_CHANGED, {
             deviceId
         });
@@ -842,6 +928,8 @@ class RealtimeClient extends RealtimeEventHandler {
    *                                               可选，视频输入设备的设备 ID。
    * @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
    *                                         可选，屏幕共享配置，如果 videoInputDeviceId 是 'screenShare'，请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
+   * @param config.prologueContent - Optional, Prologue content. | 可选，开场白内容。
+   * @param config.roomMode - Optional, Room mode. | 可选，房间模式。
    */ constructor(config){
         super(config.debug), this._client = null, this.isConnected = false, this._isTestEnv = false, this._isSupportVideo = false;
         this._config = config;

package/dist/esm/event-names/index.js

@@ -12,6 +12,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 所有服务端事件
    */ EventNames["ALL_SERVER"] = "server.*";
     /**
+   * en: Room info
+   * zh: 房间信息
+   */ EventNames["ROOM_INFO"] = "client.room.info";
+    /**
    * en: Client connected
    * zh: 客户端连接
    */ EventNames["CONNECTED"] = "client.connected";
@@ -44,6 +48,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
    * zh: 客户端视频关闭
    */ EventNames["VIDEO_OFF"] = "client.video.off";
     /**
+   * en: Client video error
+   * zh: 客户端视频(或屏幕共享）错误
+   */ EventNames["VIDEO_ERROR"] = "client.video.error";
+    /**
    * en: Client video event
    * zh: 客户端视频事件
    */ EventNames["PLAYER_EVENT"] = "client.video.event";
@@ -110,7 +118,51 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
     /**
    * en: Session updated
    * zh: 会话更新
-   */ EventNames["SESSION_UPDATE"] = "server.session.update";
+   */ EventNames["SESSION_UPDATED"] = "server.session.updated";
+    /**
+   * en: Conversation created
+   * zh: 会话创建
+   */ EventNames["CONVERSATION_CREATED"] = "server.conversation.created";
+    /**
+   * en: Conversation chat created
+   * zh: 会话对话创建
+   */ EventNames["CONVERSATION_CHAT_CREATED"] = "server.conversation.chat.created";
+    /**
+   * en: Conversation chat in progress
+   * zh: 对话正在处理中
+   */ EventNames["CONVERSATION_CHAT_IN_PROGRESS"] = "server.conversation.chat.in_progress";
+    /**
+   * en: Conversation message delta received
+   * zh: 文本消息增量返回
+   */ EventNames["CONVERSATION_MESSAGE_DELTA"] = "server.conversation.message.delta";
+    /**
+   * en: Conversation message completed
+   * zh: 文本消息完成
+   */ EventNames["CONVERSATION_MESSAGE_COMPLETED"] = "server.conversation.message.completed";
+    /**
+   * en: Conversation chat completed
+   * zh: 对话完成
+   */ EventNames["CONVERSATION_CHAT_COMPLETED"] = "server.conversation.chat.completed";
+    /**
+   * en: Conversation chat requires action
+   * zh: 对话需要插件
+   */ EventNames["CONVERSATION_CHAT_REQUIRES_ACTION"] = "server.conversation.chat.requires_action";
+    /**
+   * en: Conversation chat failed
+   * zh: 对话失败
+   */ EventNames["CONVERSATION_CHAT_FAILED"] = "server.conversation.chat.failed";
+    /**
+   * en: Session pre answer updated
+   * zh: 安抚配置更新成功
+   */ EventNames["SESSION_PRE_ANSWER_UPDATED"] = "server.session.pre_answer.updated";
+    /**
+   * en: Conversation audio transcript delta
+   * zh: 用户语音识别字幕
+   */ EventNames["CONVERSATION_AUDIO_TRANSCRIPT_DELTA"] = "server.conversation.audio_transcript.delta";
+    /**
+   * en: Mode updated
+   * zh: 更新房间模式成功
+   */ EventNames["MODE_UPDATED"] = "server.mode.updated";
     return EventNames;
 }(event_names_EventNames || {});
 /* ESM default export */ const event_names = event_names_EventNames;