@coze/realtime-api 1.2.0 → 1.2.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/event-names/index.js +9 -1
- package/dist/cjs/index.js +81 -37
- package/dist/esm/event-names/index.js +9 -1
- package/dist/esm/index.js +81 -37
- package/dist/types/client.d.ts +1 -0
- package/dist/types/event-names/client.d.ts +1 -0
- package/dist/types/event-names/event-names.d.ts +11 -1
- package/dist/types/event-names/index.d.ts +11 -2
- package/dist/types/event-names/utils.d.ts +6 -0
- package/dist/types/event-names.d.ts +11 -1
- package/dist/types/index.d.ts +11 -2
- package/dist/types/utils.d.ts +6 -0
- package/dist/umd/index.js +81 -37
- package/package.json +5 -5
@@ -49,6 +49,10 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
|
|
49
49
|
* zh: 所有服务端事件
|
50
50
|
*/ EventNames["ALL_SERVER"] = "server.*";
|
51
51
|
/**
|
52
|
+
* en: Room info
|
53
|
+
* zh: 房间信息
|
54
|
+
*/ EventNames["ROOM_INFO"] = "client.room.info";
|
55
|
+
/**
|
52
56
|
* en: Client connected
|
53
57
|
* zh: 客户端连接
|
54
58
|
*/ EventNames["CONNECTED"] = "client.connected";
|
@@ -81,6 +85,10 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
|
|
81
85
|
* zh: 客户端视频关闭
|
82
86
|
*/ EventNames["VIDEO_OFF"] = "client.video.off";
|
83
87
|
/**
|
88
|
+
* en: Client video error
|
89
|
+
* zh: 客户端视频(或屏幕共享)错误
|
90
|
+
*/ EventNames["VIDEO_ERROR"] = "client.video.error";
|
91
|
+
/**
|
84
92
|
* en: Client video event
|
85
93
|
* zh: 客户端视频事件
|
86
94
|
*/ EventNames["PLAYER_EVENT"] = "client.video.event";
|
@@ -147,7 +155,7 @@ var EventNames = /*#__PURE__*/ function(EventNames) {
|
|
147
155
|
/**
|
148
156
|
* en: Session updated
|
149
157
|
* zh: 会话更新
|
150
|
-
*/ EventNames["
|
158
|
+
*/ EventNames["SESSION_UPDATED"] = "server.session.updated";
|
151
159
|
return EventNames;
|
152
160
|
}(EventNames || {});
|
153
161
|
/* ESM default export */ const __WEBPACK_DEFAULT_EXPORT__ = EventNames;
|
package/dist/cjs/index.js
CHANGED
@@ -61,6 +61,7 @@ __webpack_require__.d(utils_namespaceObject, {
|
|
61
61
|
checkDevicePermission: ()=>checkDevicePermission,
|
62
62
|
checkPermission: ()=>checkPermission,
|
63
63
|
getAudioDevices: ()=>getAudioDevices,
|
64
|
+
isMobileVideoDevice: ()=>isMobileVideoDevice,
|
64
65
|
isScreenShareDevice: ()=>isScreenShareDevice,
|
65
66
|
isScreenShareSupported: ()=>isScreenShareSupported,
|
66
67
|
sleep: ()=>sleep
|
@@ -135,6 +136,11 @@ var rtc_default = /*#__PURE__*/ __webpack_require__.n(rtc_namespaceObject);
|
|
135
136
|
};
|
136
137
|
};
|
137
138
|
const isScreenShareDevice = (deviceId)=>'screenShare' === deviceId;
|
139
|
+
/**
|
140
|
+
* 判断是否前后置摄像头
|
141
|
+
* @param deviceId
|
142
|
+
* @returns
|
143
|
+
*/ const isMobileVideoDevice = (deviceId)=>'user' === deviceId || 'environment' === deviceId;
|
138
144
|
/**
|
139
145
|
* Check if browser supports screen sharing
|
140
146
|
* 检查浏览器是否支持屏幕共享
|
@@ -156,6 +162,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
156
162
|
* zh: 所有服务端事件
|
157
163
|
*/ EventNames["ALL_SERVER"] = "server.*";
|
158
164
|
/**
|
165
|
+
* en: Room info
|
166
|
+
* zh: 房间信息
|
167
|
+
*/ EventNames["ROOM_INFO"] = "client.room.info";
|
168
|
+
/**
|
159
169
|
* en: Client connected
|
160
170
|
* zh: 客户端连接
|
161
171
|
*/ EventNames["CONNECTED"] = "client.connected";
|
@@ -188,6 +198,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
188
198
|
* zh: 客户端视频关闭
|
189
199
|
*/ EventNames["VIDEO_OFF"] = "client.video.off";
|
190
200
|
/**
|
201
|
+
* en: Client video error
|
202
|
+
* zh: 客户端视频(或屏幕共享)错误
|
203
|
+
*/ EventNames["VIDEO_ERROR"] = "client.video.error";
|
204
|
+
/**
|
191
205
|
* en: Client video event
|
192
206
|
* zh: 客户端视频事件
|
193
207
|
*/ EventNames["PLAYER_EVENT"] = "client.video.event";
|
@@ -254,7 +268,7 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
254
268
|
/**
|
255
269
|
* en: Session updated
|
256
270
|
* zh: 会话更新
|
257
|
-
*/ EventNames["
|
271
|
+
*/ EventNames["SESSION_UPDATED"] = "server.session.updated";
|
258
272
|
return EventNames;
|
259
273
|
}(event_names_EventNames || {});
|
260
274
|
/* ESM default export */ const event_names = event_names_EventNames;
|
@@ -344,6 +358,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
344
358
|
this.engine.on(rtc_default().events.onUserLeave, this.handleUserLeave);
|
345
359
|
this.engine.on(rtc_default().events.onError, this.handleEventError);
|
346
360
|
this.engine.on(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
|
361
|
+
this.engine.on(rtc_default().events.onTrackEnded, this.handleTrackEnded);
|
347
362
|
if (this._isSupportVideo) this.engine.on(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
|
348
363
|
if (this._debug) {
|
349
364
|
this.engine.on(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -356,6 +371,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
356
371
|
this.engine.off(rtc_default().events.onUserLeave, this.handleUserLeave);
|
357
372
|
this.engine.off(rtc_default().events.onError, this.handleEventError);
|
358
373
|
this.engine.off(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
|
374
|
+
this.engine.off(rtc_default().events.onTrackEnded, this.handleTrackEnded);
|
359
375
|
if (this._isSupportVideo) this.engine.off(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
|
360
376
|
if (this._debug) {
|
361
377
|
this.engine.off(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -406,6 +422,10 @@ class EngineClient extends RealtimeEventHandler {
|
|
406
422
|
downlinkNetworkQuality
|
407
423
|
});
|
408
424
|
}
|
425
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
426
|
+
handleTrackEnded(event) {
|
427
|
+
if ((null == event ? void 0 : event.kind) === 'video') this.dispatch(event_names.VIDEO_OFF, event);
|
428
|
+
}
|
409
429
|
async joinRoom(options) {
|
410
430
|
const { token, roomId, uid, audioMutedDefault, videoOnDefault, isAutoSubscribeAudio } = options;
|
411
431
|
try {
|
@@ -439,7 +459,11 @@ class EngineClient extends RealtimeEventHandler {
|
|
439
459
|
const devices = await getAudioDevices({
|
440
460
|
video: true
|
441
461
|
});
|
442
|
-
if (-1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
462
|
+
if (!isMobileVideoDevice(deviceId) && -1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
463
|
+
this.engine.setLocalVideoPlayer(isScreenShareDevice(deviceId) ? rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN : rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN, {
|
464
|
+
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
465
|
+
userId: this._roomUserId
|
466
|
+
});
|
443
467
|
await this.changeVideoState(false);
|
444
468
|
if (isScreenShareDevice(deviceId)) {
|
445
469
|
if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) this.engine.setLocalVideoPlayer(rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN);
|
@@ -455,10 +479,6 @@ class EngineClient extends RealtimeEventHandler {
|
|
455
479
|
if (isAutoCapture) await this.engine.startVideoCapture(deviceId);
|
456
480
|
this._streamIndex = rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN;
|
457
481
|
}
|
458
|
-
this.engine.setLocalVideoPlayer(this._streamIndex, {
|
459
|
-
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
460
|
-
userId: this._roomUserId
|
461
|
-
});
|
462
482
|
}
|
463
483
|
async createLocalStream(userId, videoConfig) {
|
464
484
|
this._roomUserId = userId;
|
@@ -468,7 +488,12 @@ class EngineClient extends RealtimeEventHandler {
|
|
468
488
|
if (!devices.audioInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get audio devices');
|
469
489
|
if (this._isSupportVideo && !devices.videoInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get video devices');
|
470
490
|
await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
|
471
|
-
if (this._isSupportVideo)
|
491
|
+
if (this._isSupportVideo) try {
|
492
|
+
await this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
|
493
|
+
this.dispatch((null == videoConfig ? void 0 : videoConfig.videoOnDefault) ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
494
|
+
} catch (e) {
|
495
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
496
|
+
}
|
472
497
|
}
|
473
498
|
async disconnect() {
|
474
499
|
try {
|
@@ -491,23 +516,18 @@ class EngineClient extends RealtimeEventHandler {
|
|
491
516
|
}
|
492
517
|
}
|
493
518
|
async changeVideoState(isVideoOn) {
|
494
|
-
|
495
|
-
if (
|
496
|
-
if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
497
|
-
else {
|
498
|
-
var _this__videoConfig;
|
499
|
-
this.engine.setVideoSourceType(rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN, rtc_namespaceObject.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
500
|
-
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
501
|
-
await this.engine.publishScreen(rtc_namespaceObject.MediaType.VIDEO);
|
502
|
-
}
|
503
|
-
} else if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
519
|
+
if (isVideoOn) {
|
520
|
+
if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
504
521
|
else {
|
505
|
-
|
506
|
-
|
522
|
+
var _this__videoConfig;
|
523
|
+
this.engine.setVideoSourceType(rtc_namespaceObject.StreamIndex.STREAM_INDEX_SCREEN, rtc_namespaceObject.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
524
|
+
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
525
|
+
await this.engine.publishScreen(rtc_namespaceObject.MediaType.VIDEO);
|
507
526
|
}
|
508
|
-
}
|
509
|
-
|
510
|
-
|
527
|
+
} else if (this._streamIndex === rtc_namespaceObject.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
528
|
+
else {
|
529
|
+
await this.engine.stopScreenCapture();
|
530
|
+
await this.engine.unpublishScreen(rtc_namespaceObject.MediaType.VIDEO);
|
511
531
|
}
|
512
532
|
}
|
513
533
|
async stop() {
|
@@ -597,6 +617,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
597
617
|
this.handleEventError = this.handleEventError.bind(this);
|
598
618
|
this.handlePlayerEvent = this.handlePlayerEvent.bind(this);
|
599
619
|
this.handleNetworkQuality = this.handleNetworkQuality.bind(this);
|
620
|
+
this.handleTrackEnded = this.handleTrackEnded.bind(this);
|
600
621
|
// Debug only
|
601
622
|
this.handleLocalAudioPropertiesReport = this.handleLocalAudioPropertiesReport.bind(this);
|
602
623
|
this.handleRemoteAudioPropertiesReport = this.handleRemoteAudioPropertiesReport.bind(this);
|
@@ -620,16 +641,17 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
620
641
|
// Step1 get token
|
621
642
|
if (getRoomInfo) roomInfo = await getRoomInfo();
|
622
643
|
else {
|
623
|
-
|
624
|
-
if (this._config.
|
625
|
-
|
644
|
+
const config = {};
|
645
|
+
if (this._config.prologueContent) config.prologue_content = this._config.prologueContent;
|
646
|
+
if (void 0 !== this._config.roomMode && null !== this._config.roomMode) config.room_mode = this._config.roomMode;
|
647
|
+
if (this._config.videoConfig) {
|
648
|
+
if (isScreenShareDevice(this._config.videoConfig.videoInputDeviceId)) config.video_config = {
|
626
649
|
stream_video_type: 'screen'
|
627
|
-
}
|
628
|
-
|
629
|
-
video_config: {
|
650
|
+
};
|
651
|
+
else config.video_config = {
|
630
652
|
stream_video_type: 'main'
|
631
|
-
}
|
632
|
-
}
|
653
|
+
};
|
654
|
+
}
|
633
655
|
roomInfo = await this._api.audio.rooms.create({
|
634
656
|
bot_id: botId,
|
635
657
|
conversation_id: conversationId || void 0,
|
@@ -644,6 +666,12 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
644
666
|
this.dispatch(event_names.ERROR, error);
|
645
667
|
throw new RealtimeAPIError(error_RealtimeError.CREATE_ROOM_ERROR, error instanceof Error ? error.message : 'Unknown error', error);
|
646
668
|
}
|
669
|
+
this.dispatch(event_names.ROOM_INFO, {
|
670
|
+
roomId: roomInfo.room_id,
|
671
|
+
uid: roomInfo.uid,
|
672
|
+
token: roomInfo.token,
|
673
|
+
appId: roomInfo.app_id
|
674
|
+
});
|
647
675
|
this._isTestEnv = TEST_APP_ID === roomInfo.app_id;
|
648
676
|
// Step2 create engine
|
649
677
|
this._client = new EngineClient(roomInfo.app_id, this._config.debug, this._isTestEnv, this._isSupportVideo, this._config.videoConfig);
|
@@ -725,10 +753,14 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
725
753
|
else this.dispatch(event_names.AUDIO_MUTED, {});
|
726
754
|
}
|
727
755
|
async setVideoEnable(isEnable) {
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
756
|
+
try {
|
757
|
+
var _this__client;
|
758
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
|
759
|
+
this.dispatch(isEnable ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
760
|
+
} catch (e) {
|
761
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
762
|
+
throw e;
|
763
|
+
}
|
732
764
|
}
|
733
765
|
/**
|
734
766
|
* en: Enable audio properties reporting (debug mode only)
|
@@ -785,9 +817,19 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
785
817
|
deviceId
|
786
818
|
});
|
787
819
|
}
|
788
|
-
|
789
|
-
|
790
|
-
|
820
|
+
/**
|
821
|
+
* en: Set the video input device
|
822
|
+
*
|
823
|
+
* zh: 设置视频输入设备
|
824
|
+
*/ async setVideoInputDevice(deviceId) {
|
825
|
+
try {
|
826
|
+
var _this__client;
|
827
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
|
828
|
+
this.dispatch(event_names.VIDEO_ON, {});
|
829
|
+
} catch (e) {
|
830
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
831
|
+
throw e;
|
832
|
+
}
|
791
833
|
this.dispatch(event_names.VIDEO_INPUT_DEVICE_CHANGED, {
|
792
834
|
deviceId
|
793
835
|
});
|
@@ -842,6 +884,8 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
842
884
|
* 可选,视频输入设备的设备 ID。
|
843
885
|
* @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
|
844
886
|
* 可选,屏幕共享配置,如果 videoInputDeviceId 是 'screenShare',请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
|
887
|
+
* @param config.prologueContent - Optional, Prologue content. | 可选,开场白内容。
|
888
|
+
* @param config.roomMode - Optional, Room mode. | 可选,房间模式。
|
845
889
|
*/ constructor(config){
|
846
890
|
super(config.debug), this._client = null, this.isConnected = false, this._isTestEnv = false, this._isSupportVideo = false;
|
847
891
|
this._config = config;
|
@@ -12,6 +12,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
12
12
|
* zh: 所有服务端事件
|
13
13
|
*/ EventNames["ALL_SERVER"] = "server.*";
|
14
14
|
/**
|
15
|
+
* en: Room info
|
16
|
+
* zh: 房间信息
|
17
|
+
*/ EventNames["ROOM_INFO"] = "client.room.info";
|
18
|
+
/**
|
15
19
|
* en: Client connected
|
16
20
|
* zh: 客户端连接
|
17
21
|
*/ EventNames["CONNECTED"] = "client.connected";
|
@@ -44,6 +48,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
44
48
|
* zh: 客户端视频关闭
|
45
49
|
*/ EventNames["VIDEO_OFF"] = "client.video.off";
|
46
50
|
/**
|
51
|
+
* en: Client video error
|
52
|
+
* zh: 客户端视频(或屏幕共享)错误
|
53
|
+
*/ EventNames["VIDEO_ERROR"] = "client.video.error";
|
54
|
+
/**
|
47
55
|
* en: Client video event
|
48
56
|
* zh: 客户端视频事件
|
49
57
|
*/ EventNames["PLAYER_EVENT"] = "client.video.event";
|
@@ -110,7 +118,7 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
110
118
|
/**
|
111
119
|
* en: Session updated
|
112
120
|
* zh: 会话更新
|
113
|
-
*/ EventNames["
|
121
|
+
*/ EventNames["SESSION_UPDATED"] = "server.session.updated";
|
114
122
|
return EventNames;
|
115
123
|
}(event_names_EventNames || {});
|
116
124
|
/* ESM default export */ const event_names = event_names_EventNames;
|
package/dist/esm/index.js
CHANGED
@@ -37,6 +37,7 @@ __webpack_require__.d(utils_namespaceObject, {
|
|
37
37
|
checkDevicePermission: ()=>checkDevicePermission,
|
38
38
|
checkPermission: ()=>checkPermission,
|
39
39
|
getAudioDevices: ()=>getAudioDevices,
|
40
|
+
isMobileVideoDevice: ()=>isMobileVideoDevice,
|
40
41
|
isScreenShareDevice: ()=>isScreenShareDevice,
|
41
42
|
isScreenShareSupported: ()=>isScreenShareSupported,
|
42
43
|
sleep: ()=>sleep
|
@@ -108,6 +109,11 @@ __webpack_require__.d(utils_namespaceObject, {
|
|
108
109
|
};
|
109
110
|
};
|
110
111
|
const isScreenShareDevice = (deviceId)=>'screenShare' === deviceId;
|
112
|
+
/**
|
113
|
+
* 判断是否前后置摄像头
|
114
|
+
* @param deviceId
|
115
|
+
* @returns
|
116
|
+
*/ const isMobileVideoDevice = (deviceId)=>'user' === deviceId || 'environment' === deviceId;
|
111
117
|
/**
|
112
118
|
* Check if browser supports screen sharing
|
113
119
|
* 检查浏览器是否支持屏幕共享
|
@@ -129,6 +135,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
129
135
|
* zh: 所有服务端事件
|
130
136
|
*/ EventNames["ALL_SERVER"] = "server.*";
|
131
137
|
/**
|
138
|
+
* en: Room info
|
139
|
+
* zh: 房间信息
|
140
|
+
*/ EventNames["ROOM_INFO"] = "client.room.info";
|
141
|
+
/**
|
132
142
|
* en: Client connected
|
133
143
|
* zh: 客户端连接
|
134
144
|
*/ EventNames["CONNECTED"] = "client.connected";
|
@@ -161,6 +171,10 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
161
171
|
* zh: 客户端视频关闭
|
162
172
|
*/ EventNames["VIDEO_OFF"] = "client.video.off";
|
163
173
|
/**
|
174
|
+
* en: Client video error
|
175
|
+
* zh: 客户端视频(或屏幕共享)错误
|
176
|
+
*/ EventNames["VIDEO_ERROR"] = "client.video.error";
|
177
|
+
/**
|
164
178
|
* en: Client video event
|
165
179
|
* zh: 客户端视频事件
|
166
180
|
*/ EventNames["PLAYER_EVENT"] = "client.video.event";
|
@@ -227,7 +241,7 @@ var event_names_EventNames = /*#__PURE__*/ function(EventNames) {
|
|
227
241
|
/**
|
228
242
|
* en: Session updated
|
229
243
|
* zh: 会话更新
|
230
|
-
*/ EventNames["
|
244
|
+
*/ EventNames["SESSION_UPDATED"] = "server.session.updated";
|
231
245
|
return EventNames;
|
232
246
|
}(event_names_EventNames || {});
|
233
247
|
/* ESM default export */ const event_names = event_names_EventNames;
|
@@ -315,6 +329,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
315
329
|
this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onUserLeave, this.handleUserLeave);
|
316
330
|
this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onError, this.handleEventError);
|
317
331
|
this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onNetworkQuality, this.handleNetworkQuality);
|
332
|
+
this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onTrackEnded, this.handleTrackEnded);
|
318
333
|
if (this._isSupportVideo) this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onPlayerEvent, this.handlePlayerEvent);
|
319
334
|
if (this._debug) {
|
320
335
|
this.engine.on(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -327,6 +342,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
327
342
|
this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onUserLeave, this.handleUserLeave);
|
328
343
|
this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onError, this.handleEventError);
|
329
344
|
this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onNetworkQuality, this.handleNetworkQuality);
|
345
|
+
this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onTrackEnded, this.handleTrackEnded);
|
330
346
|
if (this._isSupportVideo) this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onPlayerEvent, this.handlePlayerEvent);
|
331
347
|
if (this._debug) {
|
332
348
|
this.engine.off(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__["default"].events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -377,6 +393,10 @@ class EngineClient extends RealtimeEventHandler {
|
|
377
393
|
downlinkNetworkQuality
|
378
394
|
});
|
379
395
|
}
|
396
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
397
|
+
handleTrackEnded(event) {
|
398
|
+
if ((null == event ? void 0 : event.kind) === 'video') this.dispatch(event_names.VIDEO_OFF, event);
|
399
|
+
}
|
380
400
|
async joinRoom(options) {
|
381
401
|
const { token, roomId, uid, audioMutedDefault, videoOnDefault, isAutoSubscribeAudio } = options;
|
382
402
|
try {
|
@@ -410,7 +430,11 @@ class EngineClient extends RealtimeEventHandler {
|
|
410
430
|
const devices = await getAudioDevices({
|
411
431
|
video: true
|
412
432
|
});
|
413
|
-
if (-1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
433
|
+
if (!isMobileVideoDevice(deviceId) && -1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
434
|
+
this.engine.setLocalVideoPlayer(isScreenShareDevice(deviceId) ? __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_SCREEN : __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN, {
|
435
|
+
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
436
|
+
userId: this._roomUserId
|
437
|
+
});
|
414
438
|
await this.changeVideoState(false);
|
415
439
|
if (isScreenShareDevice(deviceId)) {
|
416
440
|
if (this._streamIndex === __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN) this.engine.setLocalVideoPlayer(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN);
|
@@ -426,10 +450,6 @@ class EngineClient extends RealtimeEventHandler {
|
|
426
450
|
if (isAutoCapture) await this.engine.startVideoCapture(deviceId);
|
427
451
|
this._streamIndex = __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN;
|
428
452
|
}
|
429
|
-
this.engine.setLocalVideoPlayer(this._streamIndex, {
|
430
|
-
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
431
|
-
userId: this._roomUserId
|
432
|
-
});
|
433
453
|
}
|
434
454
|
async createLocalStream(userId, videoConfig) {
|
435
455
|
this._roomUserId = userId;
|
@@ -439,7 +459,12 @@ class EngineClient extends RealtimeEventHandler {
|
|
439
459
|
if (!devices.audioInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get audio devices');
|
440
460
|
if (this._isSupportVideo && !devices.videoInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get video devices');
|
441
461
|
await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
|
442
|
-
if (this._isSupportVideo)
|
462
|
+
if (this._isSupportVideo) try {
|
463
|
+
await this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
|
464
|
+
this.dispatch((null == videoConfig ? void 0 : videoConfig.videoOnDefault) ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
465
|
+
} catch (e) {
|
466
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
467
|
+
}
|
443
468
|
}
|
444
469
|
async disconnect() {
|
445
470
|
try {
|
@@ -462,23 +487,18 @@ class EngineClient extends RealtimeEventHandler {
|
|
462
487
|
}
|
463
488
|
}
|
464
489
|
async changeVideoState(isVideoOn) {
|
465
|
-
|
466
|
-
if (
|
467
|
-
if (this._streamIndex === __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
468
|
-
else {
|
469
|
-
var _this__videoConfig;
|
470
|
-
this.engine.setVideoSourceType(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_SCREEN, __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
471
|
-
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
472
|
-
await this.engine.publishScreen(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.MediaType.VIDEO);
|
473
|
-
}
|
474
|
-
} else if (this._streamIndex === __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
490
|
+
if (isVideoOn) {
|
491
|
+
if (this._streamIndex === __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
475
492
|
else {
|
476
|
-
|
477
|
-
|
493
|
+
var _this__videoConfig;
|
494
|
+
this.engine.setVideoSourceType(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_SCREEN, __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
495
|
+
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
496
|
+
await this.engine.publishScreen(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.MediaType.VIDEO);
|
478
497
|
}
|
479
|
-
}
|
480
|
-
|
481
|
-
|
498
|
+
} else if (this._streamIndex === __WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
499
|
+
else {
|
500
|
+
await this.engine.stopScreenCapture();
|
501
|
+
await this.engine.unpublishScreen(__WEBPACK_EXTERNAL_MODULE__volcengine_rtc__.MediaType.VIDEO);
|
482
502
|
}
|
483
503
|
}
|
484
504
|
async stop() {
|
@@ -568,6 +588,7 @@ class EngineClient extends RealtimeEventHandler {
|
|
568
588
|
this.handleEventError = this.handleEventError.bind(this);
|
569
589
|
this.handlePlayerEvent = this.handlePlayerEvent.bind(this);
|
570
590
|
this.handleNetworkQuality = this.handleNetworkQuality.bind(this);
|
591
|
+
this.handleTrackEnded = this.handleTrackEnded.bind(this);
|
571
592
|
// Debug only
|
572
593
|
this.handleLocalAudioPropertiesReport = this.handleLocalAudioPropertiesReport.bind(this);
|
573
594
|
this.handleRemoteAudioPropertiesReport = this.handleRemoteAudioPropertiesReport.bind(this);
|
@@ -591,16 +612,17 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
591
612
|
// Step1 get token
|
592
613
|
if (getRoomInfo) roomInfo = await getRoomInfo();
|
593
614
|
else {
|
594
|
-
|
595
|
-
if (this._config.
|
596
|
-
|
615
|
+
const config = {};
|
616
|
+
if (this._config.prologueContent) config.prologue_content = this._config.prologueContent;
|
617
|
+
if (void 0 !== this._config.roomMode && null !== this._config.roomMode) config.room_mode = this._config.roomMode;
|
618
|
+
if (this._config.videoConfig) {
|
619
|
+
if (isScreenShareDevice(this._config.videoConfig.videoInputDeviceId)) config.video_config = {
|
597
620
|
stream_video_type: 'screen'
|
598
|
-
}
|
599
|
-
|
600
|
-
video_config: {
|
621
|
+
};
|
622
|
+
else config.video_config = {
|
601
623
|
stream_video_type: 'main'
|
602
|
-
}
|
603
|
-
}
|
624
|
+
};
|
625
|
+
}
|
604
626
|
roomInfo = await this._api.audio.rooms.create({
|
605
627
|
bot_id: botId,
|
606
628
|
conversation_id: conversationId || void 0,
|
@@ -615,6 +637,12 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
615
637
|
this.dispatch(event_names.ERROR, error);
|
616
638
|
throw new RealtimeAPIError(error_RealtimeError.CREATE_ROOM_ERROR, error instanceof Error ? error.message : 'Unknown error', error);
|
617
639
|
}
|
640
|
+
this.dispatch(event_names.ROOM_INFO, {
|
641
|
+
roomId: roomInfo.room_id,
|
642
|
+
uid: roomInfo.uid,
|
643
|
+
token: roomInfo.token,
|
644
|
+
appId: roomInfo.app_id
|
645
|
+
});
|
618
646
|
this._isTestEnv = TEST_APP_ID === roomInfo.app_id;
|
619
647
|
// Step2 create engine
|
620
648
|
this._client = new EngineClient(roomInfo.app_id, this._config.debug, this._isTestEnv, this._isSupportVideo, this._config.videoConfig);
|
@@ -696,10 +724,14 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
696
724
|
else this.dispatch(event_names.AUDIO_MUTED, {});
|
697
725
|
}
|
698
726
|
async setVideoEnable(isEnable) {
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
727
|
+
try {
|
728
|
+
var _this__client;
|
729
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
|
730
|
+
this.dispatch(isEnable ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
731
|
+
} catch (e) {
|
732
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
733
|
+
throw e;
|
734
|
+
}
|
703
735
|
}
|
704
736
|
/**
|
705
737
|
* en: Enable audio properties reporting (debug mode only)
|
@@ -756,9 +788,19 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
756
788
|
deviceId
|
757
789
|
});
|
758
790
|
}
|
759
|
-
|
760
|
-
|
761
|
-
|
791
|
+
/**
|
792
|
+
* en: Set the video input device
|
793
|
+
*
|
794
|
+
* zh: 设置视频输入设备
|
795
|
+
*/ async setVideoInputDevice(deviceId) {
|
796
|
+
try {
|
797
|
+
var _this__client;
|
798
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
|
799
|
+
this.dispatch(event_names.VIDEO_ON, {});
|
800
|
+
} catch (e) {
|
801
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
802
|
+
throw e;
|
803
|
+
}
|
762
804
|
this.dispatch(event_names.VIDEO_INPUT_DEVICE_CHANGED, {
|
763
805
|
deviceId
|
764
806
|
});
|
@@ -813,6 +855,8 @@ class RealtimeClient extends RealtimeEventHandler {
|
|
813
855
|
* 可选,视频输入设备的设备 ID。
|
814
856
|
* @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
|
815
857
|
* 可选,屏幕共享配置,如果 videoInputDeviceId 是 'screenShare',请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
|
858
|
+
* @param config.prologueContent - Optional, Prologue content. | 可选,开场白内容。
|
859
|
+
* @param config.roomMode - Optional, Room mode. | 可选,房间模式。
|
816
860
|
*/ constructor(config){
|
817
861
|
super(config.debug), this._client = null, this.isConnected = false, this._isTestEnv = false, this._isSupportVideo = false;
|
818
862
|
this._config = config;
|
package/dist/types/client.d.ts
CHANGED
@@ -19,6 +19,7 @@ export declare class EngineClient extends RealtimeEventHandler {
|
|
19
19
|
handleUserLeave(event: onUserLeaveEvent): void;
|
20
20
|
handlePlayerEvent(event: unknown): void;
|
21
21
|
handleNetworkQuality(uplinkNetworkQuality: NetworkQuality, downlinkNetworkQuality: NetworkQuality): void;
|
22
|
+
handleTrackEnded(event: any): void;
|
22
23
|
joinRoom(options: {
|
23
24
|
token: string;
|
24
25
|
roomId: string;
|
@@ -19,6 +19,7 @@ export declare class EngineClient extends RealtimeEventHandler {
|
|
19
19
|
handleUserLeave(event: onUserLeaveEvent): void;
|
20
20
|
handlePlayerEvent(event: unknown): void;
|
21
21
|
handleNetworkQuality(uplinkNetworkQuality: NetworkQuality, downlinkNetworkQuality: NetworkQuality): void;
|
22
|
+
handleTrackEnded(event: any): void;
|
22
23
|
joinRoom(options: {
|
23
24
|
token: string;
|
24
25
|
roomId: string;
|
@@ -14,6 +14,11 @@ declare enum EventNames {
|
|
14
14
|
* zh: 所有服务端事件
|
15
15
|
*/
|
16
16
|
ALL_SERVER = "server.*",
|
17
|
+
/**
|
18
|
+
* en: Room info
|
19
|
+
* zh: 房间信息
|
20
|
+
*/
|
21
|
+
ROOM_INFO = "client.room.info",
|
17
22
|
/**
|
18
23
|
* en: Client connected
|
19
24
|
* zh: 客户端连接
|
@@ -54,6 +59,11 @@ declare enum EventNames {
|
|
54
59
|
* zh: 客户端视频关闭
|
55
60
|
*/
|
56
61
|
VIDEO_OFF = "client.video.off",
|
62
|
+
/**
|
63
|
+
* en: Client video error
|
64
|
+
* zh: 客户端视频(或屏幕共享)错误
|
65
|
+
*/
|
66
|
+
VIDEO_ERROR = "client.video.error",
|
57
67
|
/**
|
58
68
|
* en: Client video event
|
59
69
|
* zh: 客户端视频事件
|
@@ -138,6 +148,6 @@ declare enum EventNames {
|
|
138
148
|
* en: Session updated
|
139
149
|
* zh: 会话更新
|
140
150
|
*/
|
141
|
-
|
151
|
+
SESSION_UPDATED = "server.session.updated"
|
142
152
|
}
|
143
153
|
export default EventNames;
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import { type ScreenConfig, type AudioPropertiesConfig, type IRTCEngine } from '@volcengine/rtc';
|
2
|
-
import { type CreateRoomData, type GetToken } from '@coze/api';
|
2
|
+
import { type CreateRoomData, type GetToken, type RoomMode } from '@coze/api';
|
3
3
|
import * as RealtimeUtils from './utils';
|
4
4
|
import EventNames from './event-names';
|
5
5
|
import { RealtimeEventHandler } from './event-handler';
|
@@ -30,6 +30,8 @@ export interface RealtimeClientConfig {
|
|
30
30
|
suppressNonStationaryNoise?: boolean /** optional, Suppress non-stationary noise, defaults to false */;
|
31
31
|
videoConfig?: VideoConfig /** optional, Video configuration */;
|
32
32
|
isAutoSubscribeAudio?: boolean /** optional, Whether to automatically subscribe to bot reply audio streams, defaults to true */;
|
33
|
+
prologueContent?: string /** optional, Prologue content */;
|
34
|
+
roomMode?: RoomMode /** optional, Room mode */;
|
33
35
|
}
|
34
36
|
declare class RealtimeClient extends RealtimeEventHandler {
|
35
37
|
_config: RealtimeClientConfig;
|
@@ -80,6 +82,8 @@ declare class RealtimeClient extends RealtimeEventHandler {
|
|
80
82
|
* 可选,视频输入设备的设备 ID。
|
81
83
|
* @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
|
82
84
|
* 可选,屏幕共享配置,如果 videoInputDeviceId 是 'screenShare',请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
|
85
|
+
* @param config.prologueContent - Optional, Prologue content. | 可选,开场白内容。
|
86
|
+
* @param config.roomMode - Optional, Room mode. | 可选,房间模式。
|
83
87
|
*/
|
84
88
|
constructor(config: RealtimeClientConfig);
|
85
89
|
/**
|
@@ -143,7 +147,12 @@ declare class RealtimeClient extends RealtimeEventHandler {
|
|
143
147
|
* zh: 设置音频输出设备
|
144
148
|
*/
|
145
149
|
setAudioOutputDevice(deviceId: string): Promise<void>;
|
146
|
-
|
150
|
+
/**
|
151
|
+
* en: Set the video input device
|
152
|
+
*
|
153
|
+
* zh: 设置视频输入设备
|
154
|
+
*/
|
155
|
+
setVideoInputDevice(deviceId: string | 'screenShare' | 'user' | 'environment'): Promise<void>;
|
147
156
|
/**
|
148
157
|
* en: Get the RTC engine instance, for detail visit https://www.volcengine.com/docs/6348/104481
|
149
158
|
*
|
@@ -36,6 +36,12 @@ export declare const getAudioDevices: ({ video, }?: {
|
|
36
36
|
videoInputs: MediaDeviceInfo[];
|
37
37
|
}>;
|
38
38
|
export declare const isScreenShareDevice: (deviceId?: string) => boolean;
|
39
|
+
/**
|
40
|
+
* 判断是否前后置摄像头
|
41
|
+
* @param deviceId
|
42
|
+
* @returns
|
43
|
+
*/
|
44
|
+
export declare const isMobileVideoDevice: (deviceId?: string) => boolean;
|
39
45
|
/**
|
40
46
|
* Check if browser supports screen sharing
|
41
47
|
* 检查浏览器是否支持屏幕共享
|
@@ -14,6 +14,11 @@ declare enum EventNames {
|
|
14
14
|
* zh: 所有服务端事件
|
15
15
|
*/
|
16
16
|
ALL_SERVER = "server.*",
|
17
|
+
/**
|
18
|
+
* en: Room info
|
19
|
+
* zh: 房间信息
|
20
|
+
*/
|
21
|
+
ROOM_INFO = "client.room.info",
|
17
22
|
/**
|
18
23
|
* en: Client connected
|
19
24
|
* zh: 客户端连接
|
@@ -54,6 +59,11 @@ declare enum EventNames {
|
|
54
59
|
* zh: 客户端视频关闭
|
55
60
|
*/
|
56
61
|
VIDEO_OFF = "client.video.off",
|
62
|
+
/**
|
63
|
+
* en: Client video error
|
64
|
+
* zh: 客户端视频(或屏幕共享)错误
|
65
|
+
*/
|
66
|
+
VIDEO_ERROR = "client.video.error",
|
57
67
|
/**
|
58
68
|
* en: Client video event
|
59
69
|
* zh: 客户端视频事件
|
@@ -138,6 +148,6 @@ declare enum EventNames {
|
|
138
148
|
* en: Session updated
|
139
149
|
* zh: 会话更新
|
140
150
|
*/
|
141
|
-
|
151
|
+
SESSION_UPDATED = "server.session.updated"
|
142
152
|
}
|
143
153
|
export default EventNames;
|
package/dist/types/index.d.ts
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
import { type ScreenConfig, type AudioPropertiesConfig, type IRTCEngine } from '@volcengine/rtc';
|
2
|
-
import { type CreateRoomData, type GetToken } from '@coze/api';
|
2
|
+
import { type CreateRoomData, type GetToken, type RoomMode } from '@coze/api';
|
3
3
|
import * as RealtimeUtils from './utils';
|
4
4
|
import EventNames from './event-names';
|
5
5
|
import { RealtimeEventHandler } from './event-handler';
|
@@ -30,6 +30,8 @@ export interface RealtimeClientConfig {
|
|
30
30
|
suppressNonStationaryNoise?: boolean /** optional, Suppress non-stationary noise, defaults to false */;
|
31
31
|
videoConfig?: VideoConfig /** optional, Video configuration */;
|
32
32
|
isAutoSubscribeAudio?: boolean /** optional, Whether to automatically subscribe to bot reply audio streams, defaults to true */;
|
33
|
+
prologueContent?: string /** optional, Prologue content */;
|
34
|
+
roomMode?: RoomMode /** optional, Room mode */;
|
33
35
|
}
|
34
36
|
declare class RealtimeClient extends RealtimeEventHandler {
|
35
37
|
_config: RealtimeClientConfig;
|
@@ -80,6 +82,8 @@ declare class RealtimeClient extends RealtimeEventHandler {
|
|
80
82
|
* 可选,视频输入设备的设备 ID。
|
81
83
|
* @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
|
82
84
|
* 可选,屏幕共享配置,如果 videoInputDeviceId 是 'screenShare',请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
|
85
|
+
* @param config.prologueContent - Optional, Prologue content. | 可选,开场白内容。
|
86
|
+
* @param config.roomMode - Optional, Room mode. | 可选,房间模式。
|
83
87
|
*/
|
84
88
|
constructor(config: RealtimeClientConfig);
|
85
89
|
/**
|
@@ -143,7 +147,12 @@ declare class RealtimeClient extends RealtimeEventHandler {
|
|
143
147
|
* zh: 设置音频输出设备
|
144
148
|
*/
|
145
149
|
setAudioOutputDevice(deviceId: string): Promise<void>;
|
146
|
-
|
150
|
+
/**
|
151
|
+
* en: Set the video input device
|
152
|
+
*
|
153
|
+
* zh: 设置视频输入设备
|
154
|
+
*/
|
155
|
+
setVideoInputDevice(deviceId: string | 'screenShare' | 'user' | 'environment'): Promise<void>;
|
147
156
|
/**
|
148
157
|
* en: Get the RTC engine instance, for detail visit https://www.volcengine.com/docs/6348/104481
|
149
158
|
*
|
package/dist/types/utils.d.ts
CHANGED
@@ -36,6 +36,12 @@ export declare const getAudioDevices: ({ video, }?: {
|
|
36
36
|
videoInputs: MediaDeviceInfo[];
|
37
37
|
}>;
|
38
38
|
export declare const isScreenShareDevice: (deviceId?: string) => boolean;
|
39
|
+
/**
|
40
|
+
* 判断是否前后置摄像头
|
41
|
+
* @param deviceId
|
42
|
+
* @returns
|
43
|
+
*/
|
44
|
+
export declare const isMobileVideoDevice: (deviceId?: string) => boolean;
|
39
45
|
/**
|
40
46
|
* Check if browser supports screen sharing
|
41
47
|
* 检查浏览器是否支持屏幕共享
|
package/dist/umd/index.js
CHANGED
@@ -96,6 +96,7 @@
|
|
96
96
|
checkDevicePermission: ()=>checkDevicePermission,
|
97
97
|
checkPermission: ()=>checkPermission,
|
98
98
|
getAudioDevices: ()=>getAudioDevices,
|
99
|
+
isMobileVideoDevice: ()=>isMobileVideoDevice,
|
99
100
|
isScreenShareDevice: ()=>isScreenShareDevice,
|
100
101
|
isScreenShareSupported: ()=>isScreenShareSupported,
|
101
102
|
sleep: ()=>sleep
|
@@ -173,6 +174,11 @@
|
|
173
174
|
};
|
174
175
|
const isScreenShareDevice = (deviceId)=>'screenShare' === deviceId;
|
175
176
|
/**
|
177
|
+
* 判断是否前后置摄像头
|
178
|
+
* @param deviceId
|
179
|
+
* @returns
|
180
|
+
*/ const isMobileVideoDevice = (deviceId)=>'user' === deviceId || 'environment' === deviceId;
|
181
|
+
/**
|
176
182
|
* Check if browser supports screen sharing
|
177
183
|
* 检查浏览器是否支持屏幕共享
|
178
184
|
*/ function isScreenShareSupported() {
|
@@ -193,6 +199,10 @@
|
|
193
199
|
* zh: 所有服务端事件
|
194
200
|
*/ EventNames["ALL_SERVER"] = "server.*";
|
195
201
|
/**
|
202
|
+
* en: Room info
|
203
|
+
* zh: 房间信息
|
204
|
+
*/ EventNames["ROOM_INFO"] = "client.room.info";
|
205
|
+
/**
|
196
206
|
* en: Client connected
|
197
207
|
* zh: 客户端连接
|
198
208
|
*/ EventNames["CONNECTED"] = "client.connected";
|
@@ -225,6 +235,10 @@
|
|
225
235
|
* zh: 客户端视频关闭
|
226
236
|
*/ EventNames["VIDEO_OFF"] = "client.video.off";
|
227
237
|
/**
|
238
|
+
* en: Client video error
|
239
|
+
* zh: 客户端视频(或屏幕共享)错误
|
240
|
+
*/ EventNames["VIDEO_ERROR"] = "client.video.error";
|
241
|
+
/**
|
228
242
|
* en: Client video event
|
229
243
|
* zh: 客户端视频事件
|
230
244
|
*/ EventNames["PLAYER_EVENT"] = "client.video.event";
|
@@ -291,7 +305,7 @@
|
|
291
305
|
/**
|
292
306
|
* en: Session updated
|
293
307
|
* zh: 会话更新
|
294
|
-
*/ EventNames["
|
308
|
+
*/ EventNames["SESSION_UPDATED"] = "server.session.updated";
|
295
309
|
return EventNames;
|
296
310
|
}(event_names_EventNames || {});
|
297
311
|
/* ESM default export */ const event_names = event_names_EventNames;
|
@@ -382,6 +396,7 @@
|
|
382
396
|
this.engine.on(rtc_default().events.onUserLeave, this.handleUserLeave);
|
383
397
|
this.engine.on(rtc_default().events.onError, this.handleEventError);
|
384
398
|
this.engine.on(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
|
399
|
+
this.engine.on(rtc_default().events.onTrackEnded, this.handleTrackEnded);
|
385
400
|
if (this._isSupportVideo) this.engine.on(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
|
386
401
|
if (this._debug) {
|
387
402
|
this.engine.on(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -394,6 +409,7 @@
|
|
394
409
|
this.engine.off(rtc_default().events.onUserLeave, this.handleUserLeave);
|
395
410
|
this.engine.off(rtc_default().events.onError, this.handleEventError);
|
396
411
|
this.engine.off(rtc_default().events.onNetworkQuality, this.handleNetworkQuality);
|
412
|
+
this.engine.off(rtc_default().events.onTrackEnded, this.handleTrackEnded);
|
397
413
|
if (this._isSupportVideo) this.engine.off(rtc_default().events.onPlayerEvent, this.handlePlayerEvent);
|
398
414
|
if (this._debug) {
|
399
415
|
this.engine.off(rtc_default().events.onLocalAudioPropertiesReport, this.handleLocalAudioPropertiesReport);
|
@@ -444,6 +460,10 @@
|
|
444
460
|
downlinkNetworkQuality
|
445
461
|
});
|
446
462
|
}
|
463
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
464
|
+
handleTrackEnded(event) {
|
465
|
+
if ((null == event ? void 0 : event.kind) === 'video') this.dispatch(event_names.VIDEO_OFF, event);
|
466
|
+
}
|
447
467
|
async joinRoom(options) {
|
448
468
|
const { token, roomId, uid, audioMutedDefault, videoOnDefault, isAutoSubscribeAudio } = options;
|
449
469
|
try {
|
@@ -477,7 +497,11 @@
|
|
477
497
|
const devices = await getAudioDevices({
|
478
498
|
video: true
|
479
499
|
});
|
480
|
-
if (-1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
500
|
+
if (!isMobileVideoDevice(deviceId) && -1 === devices.videoInputs.findIndex((i)=>i.deviceId === deviceId)) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, `Video input device not found: ${deviceId}`);
|
501
|
+
this.engine.setLocalVideoPlayer(isScreenShareDevice(deviceId) ? rtc_.StreamIndex.STREAM_INDEX_SCREEN : rtc_.StreamIndex.STREAM_INDEX_MAIN, {
|
502
|
+
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
503
|
+
userId: this._roomUserId
|
504
|
+
});
|
481
505
|
await this.changeVideoState(false);
|
482
506
|
if (isScreenShareDevice(deviceId)) {
|
483
507
|
if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) this.engine.setLocalVideoPlayer(rtc_.StreamIndex.STREAM_INDEX_MAIN);
|
@@ -493,10 +517,6 @@
|
|
493
517
|
if (isAutoCapture) await this.engine.startVideoCapture(deviceId);
|
494
518
|
this._streamIndex = rtc_.StreamIndex.STREAM_INDEX_MAIN;
|
495
519
|
}
|
496
|
-
this.engine.setLocalVideoPlayer(this._streamIndex, {
|
497
|
-
renderDom: (null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.renderDom) || 'local-player',
|
498
|
-
userId: this._roomUserId
|
499
|
-
});
|
500
520
|
}
|
501
521
|
async createLocalStream(userId, videoConfig) {
|
502
522
|
this._roomUserId = userId;
|
@@ -506,7 +526,12 @@
|
|
506
526
|
if (!devices.audioInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get audio devices');
|
507
527
|
if (this._isSupportVideo && !devices.videoInputs.length) throw new RealtimeAPIError(error_RealtimeError.DEVICE_ACCESS_ERROR, 'Failed to get video devices');
|
508
528
|
await this.engine.startAudioCapture(devices.audioInputs[0].deviceId);
|
509
|
-
if (this._isSupportVideo)
|
529
|
+
if (this._isSupportVideo) try {
|
530
|
+
await this.setVideoInputDevice((null == videoConfig ? void 0 : videoConfig.videoInputDeviceId) || devices.videoInputs[0].deviceId, null == videoConfig ? void 0 : videoConfig.videoOnDefault);
|
531
|
+
this.dispatch((null == videoConfig ? void 0 : videoConfig.videoOnDefault) ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
532
|
+
} catch (e) {
|
533
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
534
|
+
}
|
510
535
|
}
|
511
536
|
async disconnect() {
|
512
537
|
try {
|
@@ -529,23 +554,18 @@
|
|
529
554
|
}
|
530
555
|
}
|
531
556
|
async changeVideoState(isVideoOn) {
|
532
|
-
|
533
|
-
if (
|
534
|
-
if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
535
|
-
else {
|
536
|
-
var _this__videoConfig;
|
537
|
-
this.engine.setVideoSourceType(rtc_.StreamIndex.STREAM_INDEX_SCREEN, rtc_.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
538
|
-
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
539
|
-
await this.engine.publishScreen(rtc_.MediaType.VIDEO);
|
540
|
-
}
|
541
|
-
} else if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
557
|
+
if (isVideoOn) {
|
558
|
+
if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.startVideoCapture();
|
542
559
|
else {
|
543
|
-
|
544
|
-
|
560
|
+
var _this__videoConfig;
|
561
|
+
this.engine.setVideoSourceType(rtc_.StreamIndex.STREAM_INDEX_SCREEN, rtc_.VideoSourceType.VIDEO_SOURCE_TYPE_INTERNAL);
|
562
|
+
await this.engine.startScreenCapture(null === (_this__videoConfig = this._videoConfig) || void 0 === _this__videoConfig ? void 0 : _this__videoConfig.screenConfig);
|
563
|
+
await this.engine.publishScreen(rtc_.MediaType.VIDEO);
|
545
564
|
}
|
546
|
-
}
|
547
|
-
|
548
|
-
|
565
|
+
} else if (this._streamIndex === rtc_.StreamIndex.STREAM_INDEX_MAIN) await this.engine.stopVideoCapture();
|
566
|
+
else {
|
567
|
+
await this.engine.stopScreenCapture();
|
568
|
+
await this.engine.unpublishScreen(rtc_.MediaType.VIDEO);
|
549
569
|
}
|
550
570
|
}
|
551
571
|
async stop() {
|
@@ -635,6 +655,7 @@
|
|
635
655
|
this.handleEventError = this.handleEventError.bind(this);
|
636
656
|
this.handlePlayerEvent = this.handlePlayerEvent.bind(this);
|
637
657
|
this.handleNetworkQuality = this.handleNetworkQuality.bind(this);
|
658
|
+
this.handleTrackEnded = this.handleTrackEnded.bind(this);
|
638
659
|
// Debug only
|
639
660
|
this.handleLocalAudioPropertiesReport = this.handleLocalAudioPropertiesReport.bind(this);
|
640
661
|
this.handleRemoteAudioPropertiesReport = this.handleRemoteAudioPropertiesReport.bind(this);
|
@@ -658,16 +679,17 @@
|
|
658
679
|
// Step1 get token
|
659
680
|
if (getRoomInfo) roomInfo = await getRoomInfo();
|
660
681
|
else {
|
661
|
-
|
662
|
-
if (this._config.
|
663
|
-
|
682
|
+
const config = {};
|
683
|
+
if (this._config.prologueContent) config.prologue_content = this._config.prologueContent;
|
684
|
+
if (void 0 !== this._config.roomMode && null !== this._config.roomMode) config.room_mode = this._config.roomMode;
|
685
|
+
if (this._config.videoConfig) {
|
686
|
+
if (isScreenShareDevice(this._config.videoConfig.videoInputDeviceId)) config.video_config = {
|
664
687
|
stream_video_type: 'screen'
|
665
|
-
}
|
666
|
-
|
667
|
-
video_config: {
|
688
|
+
};
|
689
|
+
else config.video_config = {
|
668
690
|
stream_video_type: 'main'
|
669
|
-
}
|
670
|
-
}
|
691
|
+
};
|
692
|
+
}
|
671
693
|
roomInfo = await this._api.audio.rooms.create({
|
672
694
|
bot_id: botId,
|
673
695
|
conversation_id: conversationId || void 0,
|
@@ -682,6 +704,12 @@
|
|
682
704
|
this.dispatch(event_names.ERROR, error);
|
683
705
|
throw new RealtimeAPIError(error_RealtimeError.CREATE_ROOM_ERROR, error instanceof Error ? error.message : 'Unknown error', error);
|
684
706
|
}
|
707
|
+
this.dispatch(event_names.ROOM_INFO, {
|
708
|
+
roomId: roomInfo.room_id,
|
709
|
+
uid: roomInfo.uid,
|
710
|
+
token: roomInfo.token,
|
711
|
+
appId: roomInfo.app_id
|
712
|
+
});
|
685
713
|
this._isTestEnv = TEST_APP_ID === roomInfo.app_id;
|
686
714
|
// Step2 create engine
|
687
715
|
this._client = new EngineClient(roomInfo.app_id, this._config.debug, this._isTestEnv, this._isSupportVideo, this._config.videoConfig);
|
@@ -763,10 +791,14 @@
|
|
763
791
|
else this.dispatch(event_names.AUDIO_MUTED, {});
|
764
792
|
}
|
765
793
|
async setVideoEnable(isEnable) {
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
794
|
+
try {
|
795
|
+
var _this__client;
|
796
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.changeVideoState(isEnable));
|
797
|
+
this.dispatch(isEnable ? event_names.VIDEO_ON : event_names.VIDEO_OFF, {});
|
798
|
+
} catch (e) {
|
799
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
800
|
+
throw e;
|
801
|
+
}
|
770
802
|
}
|
771
803
|
/**
|
772
804
|
* en: Enable audio properties reporting (debug mode only)
|
@@ -823,9 +855,19 @@
|
|
823
855
|
deviceId
|
824
856
|
});
|
825
857
|
}
|
826
|
-
|
827
|
-
|
828
|
-
|
858
|
+
/**
|
859
|
+
* en: Set the video input device
|
860
|
+
*
|
861
|
+
* zh: 设置视频输入设备
|
862
|
+
*/ async setVideoInputDevice(deviceId) {
|
863
|
+
try {
|
864
|
+
var _this__client;
|
865
|
+
await (null === (_this__client = this._client) || void 0 === _this__client ? void 0 : _this__client.setVideoInputDevice(deviceId));
|
866
|
+
this.dispatch(event_names.VIDEO_ON, {});
|
867
|
+
} catch (e) {
|
868
|
+
this.dispatch(event_names.VIDEO_ERROR, e);
|
869
|
+
throw e;
|
870
|
+
}
|
829
871
|
this.dispatch(event_names.VIDEO_INPUT_DEVICE_CHANGED, {
|
830
872
|
deviceId
|
831
873
|
});
|
@@ -880,6 +922,8 @@
|
|
880
922
|
* 可选,视频输入设备的设备 ID。
|
881
923
|
* @param config.videoConfig.screenConfig - Optional, Screen share configuration if videoInputDeviceId is 'screenShare' see https://www.volcengine.com/docs/6348/104481#screenconfig for more details. |
|
882
924
|
* 可选,屏幕共享配置,如果 videoInputDeviceId 是 'screenShare',请参考 https://www.volcengine.com/docs/6348/104481#screenconfig 了解更多详情。
|
925
|
+
* @param config.prologueContent - Optional, Prologue content. | 可选,开场白内容。
|
926
|
+
* @param config.roomMode - Optional, Room mode. | 可选,房间模式。
|
883
927
|
*/ constructor(config){
|
884
928
|
super(config.debug), this._client = null, this.isConnected = false, this._isTestEnv = false, this._isSupportVideo = false;
|
885
929
|
this._config = config;
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@coze/realtime-api",
|
3
|
-
"version": "1.2.
|
3
|
+
"version": "1.2.1-beta.10",
|
4
4
|
"description": "A powerful real-time communication SDK for voice interactions with Coze AI bots | 扣子官方实时通信 SDK,用于与 Coze AI bots 进行语音交互",
|
5
5
|
"keywords": [
|
6
6
|
"coze",
|
@@ -54,8 +54,8 @@
|
|
54
54
|
"test:cov": "vitest --coverage --run"
|
55
55
|
},
|
56
56
|
"dependencies": {
|
57
|
-
"@coze/api": "1.2.
|
58
|
-
"@volcengine/rtc": "
|
57
|
+
"@coze/api": "1.2.1-beta.20",
|
58
|
+
"@volcengine/rtc": "~4.62.11"
|
59
59
|
},
|
60
60
|
"devDependencies": {
|
61
61
|
"@coze-infra/eslint-config": "workspace:*",
|
@@ -66,10 +66,10 @@
|
|
66
66
|
"@types/node": "^20",
|
67
67
|
"@types/uuid": "^9.0.1",
|
68
68
|
"@types/whatwg-fetch": "^0.0.33",
|
69
|
-
"@vitest/coverage-v8": "~2.1.
|
69
|
+
"@vitest/coverage-v8": "~2.1.9",
|
70
70
|
"axios": "^1.7.7",
|
71
71
|
"typescript": "^5.5.3",
|
72
|
-
"vitest": "~2.1.
|
72
|
+
"vitest": "~2.1.9"
|
73
73
|
},
|
74
74
|
"cozePublishConfig": {
|
75
75
|
"exports": {
|