@tencent-rtc/mcp 1.3.4 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +4 -4
- package/index.js.map +1 -1
- package/package.json +1 -1
- package/resource/doc/chatuikit/android/features/theme.md +363 -0
- package/resource/doc/chatuikit/{android/client_api_Android.md → faq/android_sdk/client-api.md} +0 -2
- package/resource/doc/chatuikit/faq/common/product.md +8 -0
- package/resource/doc/chatuikit/faq/flutter_sdk/faq.md +53 -0
- package/resource/doc/chatuikit/{iOS/client_api_iOS.md → faq/ios_sdk/client-api.md} +0 -2
- package/resource/doc/chatuikit/flutter/features/theme.md +379 -0
- package/resource/doc/chatuikit/flutter/integration.md +413 -0
- package/resource/doc/chatuikit/flutter/workflow.md +941 -0
- package/resource/doc/chatuikit/iOS/features/theme.md +416 -0
- package/resource/doc/chatuikit/react/components/chat-setting.md +86 -0
- package/resource/doc/chatuikit/react/components/contact-list.md +1093 -0
- package/resource/doc/chatuikit/react/components/search.md +745 -0
- package/resource/doc/chatuikit/vue/components/chat-setting.md +64 -0
- package/resource/doc/chatuikit/vue/components/contact-list.md +679 -0
- package/resource/doc/chatuikit/vue/components/search.md +1384 -0
- package/resource/doc/livekit/live/android/workflow.md +137 -6
- package/resource/doc/livekit/live/ios/live-audience-list.md +339 -0
- package/resource/doc/livekit/live/ios/live-audio-effect.md +370 -0
- package/resource/doc/livekit/live/ios/live-barrage.md +517 -0
- package/resource/doc/livekit/live/ios/live-base-beauty.md +247 -0
- package/resource/doc/livekit/live/ios/live-co-guest-interaction.md +518 -0
- package/resource/doc/livekit/live/ios/live-gift.md +783 -0
- package/resource/doc/livekit/live/ios/live-host-interaction.md +815 -0
- package/resource/doc/livekit/live/ios/live-integration.md +931 -0
- package/resource/doc/livekit/live/ios/live-live-list.md +485 -0
- package/resource/doc/livekit/live/ios/workflow.md +454 -0
- package/resource/doc/rtcengine/web/plugin/20-advanced-customized-capture-rendering.md +87 -0
- package/resource/doc/rtcengine/web/plugin/21-advanced-auto-play-policy.md +87 -0
- package/resource/doc/rtcengine/web/plugin/22-advanced-audio-mixer.md +175 -0
- package/resource/doc/rtcengine/web/plugin/23-advanced-support-detection.md +221 -0
- package/resource/doc/rtcengine/web/plugin/24-advanced-network-quality.md +140 -0
- package/resource/doc/rtcengine/web/plugin/25-advanced-device-change.md +99 -0
- package/resource/doc/rtcengine/web/plugin/26-advanced-publish-cdn-stream.md +451 -0
- package/resource/doc/rtcengine/web/plugin/27-advanced-small-stream.md +151 -0
- package/resource/doc/rtcengine/web/plugin/28-advanced-beauty.md +322 -0
- package/resource/doc/rtcengine/web/plugin/29-advanced-water-mark.md +149 -0
- package/resource/doc/rtcengine/web/plugin/30-advanced-cross-room-link.md +96 -0
- package/resource/doc/rtcengine/web/plugin/31-advanced-plugins-overview.md +330 -0
- package/resource/doc/rtcengine/web/plugin/34-advanced-proxy.md +248 -0
- package/resource/doc/rtcengine/web/plugin/35-advanced-ai-denoiser.md +81 -0
- package/resource/doc/rtcengine/web/plugin/36-advanced-virtual-background.md +242 -0
- package/resource/doc/rtcengine/web/plugin/37-advanced-voice-changer.md +99 -0
- package/resource/doc/rtcengine/web/plugin/39-advanced-video-decoder.md +33 -0
- package/resource/doc/rtcengine/web/plugin/40-advanced-video-mixer.md +379 -0
- package/resource/doc/rtcengine/web/plugin/41-advanced-small-stream-auto-switcher.md +91 -0
- package/resource/doc/rtcengine/web/plugin/42-advanced-realtime-transcriber.md +315 -0
- package/resource/doc/rtcengine/web/plugin/43-advanced-face-detection.md +180 -0
- package/resource/doc/tuicallkit/android/workflow.md +0 -0
- package/resource/doc/tuicallkit/flutter/workflow.md +331 -0
- package/tools/call/get-native-call-integration.js +27 -11
- package/tools/call/get-native-call-integration.js.map +1 -1
- package/tools/chat/get-native-chat-feature-detail.js +30 -0
- package/tools/chat/get-native-chat-feature-detail.js.map +1 -0
- package/tools/chat/get-native-chat-integration.js +30 -12
- package/tools/chat/get-native-chat-integration.js.map +1 -1
- package/tools/common/get-faq.js +2 -4
- package/tools/common/get-faq.js.map +1 -1
- package/tools/definitions/get_native_call_uikit_integration_def.js +17 -7
- package/tools/definitions/get_native_call_uikit_integration_def.js.map +1 -1
- package/tools/definitions/get_native_chat_feature_detail_def.js +32 -0
- package/tools/definitions/get_native_chat_feature_detail_def.js.map +1 -0
- package/tools/definitions/get_native_chat_uikit_integration_def.js +19 -7
- package/tools/definitions/get_native_chat_uikit_integration_def.js.map +1 -1
- package/tools/definitions/{get_native_live_extra_features_def.js → get_native_live_uikit_extra_features_def.js} +3 -3
- package/tools/definitions/get_native_live_uikit_extra_features_def.js.map +1 -0
- package/tools/definitions/{get_native_live_interaction_def.js → get_native_live_uikit_interaction_def.js} +3 -3
- package/tools/definitions/get_native_live_uikit_interaction_def.js.map +1 -0
- package/tools/definitions/get_web_chat_uikit_component_detail_def.js +49 -10
- package/tools/definitions/get_web_chat_uikit_component_detail_def.js.map +1 -1
- package/tools/index.js +6 -4
- package/tools/index.js.map +1 -1
- package/tools/live/{get_native_live_extra_features.js → get_native_live_uikit_extra_features.js} +4 -4
- package/tools/live/get_native_live_uikit_extra_features.js.map +1 -0
- package/tools/live/{get_native_live_interaction.js → get_native_live_uikit_interaction.js} +4 -4
- package/tools/live/get_native_live_uikit_interaction.js.map +1 -0
- package/tools/rtcengine/get-rtcengine-web-sdk-integration.js.map +1 -1
- package/tools/rtcengine/get-trtc-sdk-faq.js +32 -0
- package/tools/rtcengine/get-trtc-sdk-faq.js.map +1 -0
- package/tools/rtcengine/get-trtc-web-sdk-faq.js +1 -1
- package/tools/rtcengine/get-trtc-web-sdk-integration.js.map +1 -1
- package/tools/rtcengine/get-trtc-web-sdk-plugin.js +45 -0
- package/tools/rtcengine/get-trtc-web-sdk-plugin.js.map +1 -0
- package/utils/handle-component-name.js +0 -16
- package/utils/handle-component-name.js.map +1 -1
- package/utils/report-cls-client.js +1 -1
- package/utils/report-es-client.js +1 -1
- package/resource/doc/chatuikit/android/generate_UserSig.md +0 -306
- package/resource/doc/chatuikit/android/integrarte_only_tuichat_Android.md +0 -290
- package/resource/doc/chatuikit/android/quick_start_Android.md +0 -138
- package/resource/doc/chatuikit/android/show_typing_Android.md +0 -43
- package/resource/doc/chatuikit/flutter/generate_UserSig.md +0 -306
- package/resource/doc/chatuikit/flutter/integrate_tuikit_Flutter.md +0 -133
- package/resource/doc/chatuikit/iOS/generate_UserSig.md +0 -306
- package/resource/doc/chatuikit/iOS/integrate_only_tuichat_iOS.md +0 -748
- package/resource/doc/chatuikit/iOS/quick_start_iOS.md +0 -163
- package/resource/doc/chatuikit/iOS/show_typing_iOS.md +0 -52
- package/resource/doc/tuicallkit/flutter/how_to_run_flutter_callkit_demo.md +0 -101
- package/resource/doc/tuicallkit/iOS/how_to_run_iOS_callkit_demo.md +0 -98
- package/tools/definitions/get_native_live_extra_features_def.js.map +0 -1
- package/tools/definitions/get_native_live_interaction_def.js.map +0 -1
- package/tools/live/get_native_live_extra_features.js.map +0 -1
- package/tools/live/get_native_live_interaction.js.map +0 -1
- /package/resource/doc/chatuikit/android/{integrate_tuikit_Android.md → integration.md} +0 -0
- /package/resource/doc/chatuikit/{faq/native_sdk/faq.md → android/workflow.md} +0 -0
- /package/resource/doc/chatuikit/{flutter/client_api_Flutter.md → faq/flutter_sdk/client-api.md} +0 -0
- /package/resource/doc/chatuikit/iOS/{integrate_tuikit_iOS.md → integration.md} +0 -0
- /package/resource/doc/{tuicallkit/android/.gitkeep → chatuikit/iOS/workflow.md} +0 -0
- /package/resource/doc/chatuikit/react/components/{chatheader.md → chat-header.md} +0 -0
- /package/resource/doc/chatuikit/vue/components/{chatheader.md → chat-header.md} +0 -0
- /package/resource/doc/tuicallkit/android/{how_to_integrated_android_callkit.md → integration.md} +0 -0
- /package/resource/doc/tuicallkit/{android/api_TUICallObserver.md → faq/android_sdk/call-observer-api.md} +0 -0
- /package/resource/doc/tuicallkit/{android/api_TUICallKit.md → faq/android_sdk/callkit-api.md} +0 -0
- /package/resource/doc/tuicallkit/{flutter/api_TUICallDefine.md → faq/flutter_sdk/call-define-api.md} +0 -0
- /package/resource/doc/tuicallkit/{flutter/api_TUICallObserver.md → faq/flutter_sdk/call-observer-api.md} +0 -0
- /package/resource/doc/tuicallkit/{flutter/api_TUICallKit.md → faq/flutter_sdk/callkit-api.md} +0 -0
- /package/resource/doc/tuicallkit/{iOS/api_TUICallObserver.md → faq/ios_sdk/call-observer-api.md} +0 -0
- /package/resource/doc/tuicallkit/{iOS/api_TUICallKit.md → faq/ios_sdk/callkit-api.md} +0 -0
- /package/resource/doc/tuicallkit/flutter/{how_to_integrated_flutter_callkit.md → integration.md} +0 -0
- /package/resource/doc/tuicallkit/iOS/{how_to_integrated_iOS_callkit.md → integration.md} +0 -0
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
## Overview
|
|
2
|
+
|
|
3
|
+
This document provides a detailed guide on how to use the TRTC Web SDK's VideoMixer plugin to composite multiple input sources including camera, screen sharing, text, images, and videos into a single video track for streaming with custom layouts. Developers can flexibly control the layout (position, size, layer zIndex) of each source.
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
## Prerequisites
|
|
8
|
+
|
|
9
|
+
- TRTC Web SDK version >= 5.12.0.
|
|
10
|
+
|
|
11
|
+
- Compatibility:
|
|
12
|
+
|
|
13
|
+
| Browser | Compatibility |
|
|
14
|
+
| ---------- | ------------------------------ |
|
|
15
|
+
| PC Chrome | ✅ |
|
|
16
|
+
| PC Safari | ✅ |
|
|
17
|
+
| PC Firefox | ✅ |
|
|
18
|
+
| PC Edge | ✅ |
|
|
19
|
+
| Mobile | ❌ Mixing not supported yet |
|
|
20
|
+
|
|
21
|
+
## Implementation
|
|
22
|
+
|
|
23
|
+
### 1. Import and Register Plugin
|
|
24
|
+
|
|
25
|
+
```javascript
|
|
26
|
+
import { VideoMixer } from 'trtc-sdk-v5/plugins/video-effect/video-mixer';
|
|
27
|
+
|
|
28
|
+
const trtc = TRTC.create({ plugins: [VideoMixer] });
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### 2. Start Mixing
|
|
32
|
+
|
|
33
|
+
Start the mixing plugin by calling `trtc.startPlugin('VideoMixer', options)`. You must set the canvas width and height [canvasInfo](#canvasinfo) in options, all other parameters are optional.
|
|
34
|
+
|
|
35
|
+
```javascript
|
|
36
|
+
const options = {
|
|
37
|
+
view: 'local_preview_div', // Preview
|
|
38
|
+
canvasInfo: { width: 1920, height: 1080 },
|
|
39
|
+
camera: [{ // Add a camera input source
|
|
40
|
+
id: 'camera1',
|
|
41
|
+
layout: { x: 0, y: 0, width: 640, height: 480, zIndex: 1 }
|
|
42
|
+
}]
|
|
43
|
+
// ...
|
|
44
|
+
}
|
|
45
|
+
const { track } = await trtc.startPlugin('VideoMixer', options);
|
|
46
|
+
// track is the mixed output MediaStreamTrack, which can be used for streaming
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### 3. Publish After Mixing
|
|
50
|
+
|
|
51
|
+
The mixing plugin is only responsible for generating the mixed video track. Publishing operation is still controlled by the TRTC SDK. Pass the track returned by startPlugin as a parameter to the startLocalVideo interface to publish.
|
|
52
|
+
|
|
53
|
+
```js
|
|
54
|
+
await trtc.startLocalVideo({ option: {
|
|
55
|
+
videoTrack: track,
|
|
56
|
+
profile: { width: 1920, height: 1080, bitrate: 2000 } // Set encoding parameters
|
|
57
|
+
}});
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### 4. Manage Input Sources (Update and Remove)
|
|
61
|
+
|
|
62
|
+
Dynamically add, update, or remove mixing input sources by calling `trtc.updatePlugin('VideoMixer', options)`.
|
|
63
|
+
|
|
64
|
+
**Update Rules:**
|
|
65
|
+
|
|
66
|
+
* **Input Source Description:** Input sources are divided into: `camera, screen, text, image, video`. All are arrays, each array element represents an input source, uniquely identified by the id attribute.
|
|
67
|
+
* **Update Input Source:** Pass the same id and modify other parameters.
|
|
68
|
+
* **Add Input Source:** Pass a new id object.
|
|
69
|
+
* **Remove Input Source:** Remove the object from the corresponding input source array.
|
|
70
|
+
|
|
71
|
+
#### Camera
|
|
72
|
+
|
|
73
|
+
Control camera capture and layout.
|
|
74
|
+
|
|
75
|
+
```javascript
|
|
76
|
+
// Update camera layout with id 'camera1'
|
|
77
|
+
await trtc.updatePlugin('VideoMixer', {
|
|
78
|
+
camera: [
|
|
79
|
+
{
|
|
80
|
+
id: 'camera1',
|
|
81
|
+
profile: { width: 640, height: 480, frameRate: 15 }, // Update camera capture parameters
|
|
82
|
+
layout: { x: 100, y: 0, width: 640, height: 480, zIndex: 1, rotation: 90, mirror: true } // Update camera layout parameters
|
|
83
|
+
}
|
|
84
|
+
]
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// Remove all cameras
|
|
88
|
+
await trtc.updatePlugin('VideoMixer', { camera: [] });
|
|
89
|
+
// If you only want to temporarily hide the input source on the canvas without physically closing capture, you can set hidden: true in the layout parameter. The same applies to other input sources.
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
#### Screen Sharing
|
|
93
|
+
|
|
94
|
+
```js
|
|
95
|
+
// Add screen sharing
|
|
96
|
+
await trtc.updatePlugin('VideoMixer', {
|
|
97
|
+
screen: [{
|
|
98
|
+
id: 'screen1',
|
|
99
|
+
profile: { width: 1920, height: 1080, frameRate: 15 }, // Screen sharing capture parameters
|
|
100
|
+
layout: {
|
|
101
|
+
x: 0,
|
|
102
|
+
y: 0,
|
|
103
|
+
width: 1920,
|
|
104
|
+
height: 1080,
|
|
105
|
+
zIndex: 0,
|
|
106
|
+
}
|
|
107
|
+
}],
|
|
108
|
+
// If the user stops screen sharing via browser button instead of SDK, you can listen through this callback
|
|
109
|
+
onScreenShareStop: (id: string) => console.log(`Screen sharing stopped, id: ${id}`);
|
|
110
|
+
});
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
#### Text
|
|
114
|
+
|
|
115
|
+
Add text input sources by passing an object with a unique id in the `text` parameter array to represent a text source in the mix:
|
|
116
|
+
|
|
117
|
+
> Note: When text content exceeds the layout area, the overflowing part will be clipped. You need to adjust the layout width and height according to text content.
|
|
118
|
+
|
|
119
|
+
```js
|
|
120
|
+
await trtc.updatePlugin('VideoMixer', {
|
|
121
|
+
text: [
|
|
122
|
+
{
|
|
123
|
+
id: 'text1',
|
|
124
|
+
content: 'MultiLine\nTest',
|
|
125
|
+
font: 'bold 60px SimHei',
|
|
126
|
+
color: 'red',
|
|
127
|
+
layout: {
|
|
128
|
+
x: 200,
|
|
129
|
+
y: 300,
|
|
130
|
+
width: 300,
|
|
131
|
+
height: 150,
|
|
132
|
+
zIndex: 6,
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
]
|
|
136
|
+
});
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
#### Images and Videos
|
|
140
|
+
|
|
141
|
+
Add images (such as logos) or local video files as mixing input sources.
|
|
142
|
+
|
|
143
|
+
```js
|
|
144
|
+
await trtc.updatePlugin('VideoMixer', {
|
|
145
|
+
image: [{
|
|
146
|
+
id: 'img1',
|
|
147
|
+
url: './image.png',
|
|
148
|
+
layout: { x: 0, y: 500, width: 800, height: 400, zIndex: 4 }
|
|
149
|
+
}],
|
|
150
|
+
video: [{
|
|
151
|
+
id: 'video1',
|
|
152
|
+
url: './video.mp4',
|
|
153
|
+
layout: { x: 0, y: 0, width: 1000, height: 500, zIndex: 5 }
|
|
154
|
+
}]
|
|
155
|
+
});
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### 5. Stop Mixing
|
|
159
|
+
|
|
160
|
+
Remove all input sources and stop mixing:
|
|
161
|
+
|
|
162
|
+
```javascript
|
|
163
|
+
await trtc.stopPlugin('VideoMixer');
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Error Handling
|
|
167
|
+
|
|
168
|
+
The error handling strategy of the mixing plugin is designed to ensure that errors in some input sources do not affect the mixing of other normal input sources.
|
|
169
|
+
|
|
170
|
+
| Scenario | Interface Behavior | Error Information Return Method |
|
|
171
|
+
| :---- | :---- | :---- |
|
|
172
|
+
| **Partial Success, Partial Failure** | Interface does not throw errors (try...catch cannot catch) | Failure information is notified through result.failedDetails in the return value MixParseResult |
|
|
173
|
+
| **All Failed or Critical Error** | Interface throws error (try...catch can catch) | Error object carries failure information, such as error?.data?.failedDetails |
|
|
174
|
+
|
|
175
|
+
Business logic example:
|
|
176
|
+
|
|
177
|
+
```ts
|
|
178
|
+
try {
|
|
179
|
+
const { result } = await trtc.updatePlugin('VideoMixer', {
|
|
180
|
+
camera: [/* ... */],
|
|
181
|
+
screen: [/* ... */],
|
|
182
|
+
text: [/* ... */],
|
|
183
|
+
image: [/* ... */],
|
|
184
|
+
video: [/* ... */],
|
|
185
|
+
});
|
|
186
|
+
// Scenario 1: Error handling for partial failures
|
|
187
|
+
result.failedDetails.forEach(({id, error}: {id: string, error: any}) => {
|
|
188
|
+
console.error(`${id} mix failed`, error);
|
|
189
|
+
})
|
|
190
|
+
console.log(result.successOptions) // Successfully applied options for this update
|
|
191
|
+
} catch (error: any) {
|
|
192
|
+
// Scenario 2: Error catching for all failures (catching thrown errors)
|
|
193
|
+
console.error(error);
|
|
194
|
+
error?.data?.failedDetails.forEach(({ id, error }: { id: string, error: any }) => {
|
|
195
|
+
console.error(`${id} mix failed`, error);
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
**Detailed Explanation of successOptions**
|
|
201
|
+
|
|
202
|
+
This return value informs the business side of the parameters actually applied by the mixing underlying layer after this update. When some updates fail, successOptions will include the successfully updated parts and the failed parts that retain the old configuration, making it convenient for the business side to synchronize UI or data status.
|
|
203
|
+
|
|
204
|
+
## API Reference
|
|
205
|
+
|
|
206
|
+
### trtc.startPlugin('VideoMixer', options)
|
|
207
|
+
|
|
208
|
+
Used to start the mixing plugin and initialize canvas and input sources.
|
|
209
|
+
|
|
210
|
+
<a name="options"></a>**Options**
|
|
211
|
+
|
|
212
|
+
| Name | Type | Required | Description |
|
|
213
|
+
| ----------------- | ----------------------------------- | -------- | -------------------------------------------------------------------------------------------------------------------- |
|
|
214
|
+
| view | string \| HTMLElement \| null | Optional | HTMLElement instance or Id for mixed video preview. If not passed or passed as null, the mixed video will not be played. |
|
|
215
|
+
| canvasInfo | [CanvasInfo](#canvasinfo) | Required | Parameters for setting the mixing canvas |
|
|
216
|
+
| camera | [CameraSource](#camerasource)[] | Optional | Parameters for controlling camera input sources |
|
|
217
|
+
| screen | [ScreenSource](#screensource)[] | Optional | Parameters for controlling screen sharing input sources |
|
|
218
|
+
| text | [TextSource](#textsource)[] | Optional | Parameters for controlling text input sources |
|
|
219
|
+
| image | [ImageSource](#imagesource)[] | Optional | Parameters for controlling image input sources |
|
|
220
|
+
| video | [VideoSource](#videosource)[] | Optional | Parameters for controlling video input sources |
|
|
221
|
+
| onScreenShareStop | (id: string) => {} | Optional | Callback invoked when mixed screen sharing stops. May be used when user cancels screen sharing via browser button instead of SDK. The callback parameter is the id of the stopped screen capture |
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
**Returns: Promise\<MixParseResult\>**
|
|
225
|
+
|
|
226
|
+
`MixParseResult`
|
|
227
|
+
|
|
228
|
+
| Name | Type | Description |
|
|
229
|
+
| ------ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
230
|
+
| track | [MediaStreamTrack](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) | Mixed output video track |
|
|
231
|
+
| systemAudioTrackList | object | This returns a list of system audio tracks. After setting the ScreenSource parameter `systemAudio:true` and successfully capturing the audio, it will return the system audio tracks. The data format is: `{key: audioTrack}`, where `key` is the ID configured in the ScreenSource, for example: `{id1: audioTrack, id2: audioTrack}`. |
|
|
232
|
+
| result | object | Information about the success or failure of the parameters passed in this time, structured as follows:<table><tr><th>Name</th><th>Type</th><th>Description</th></tr><tr><td>successOptions</td><td>[Options](#options)</td><td>Successfully executed options for this update</td></tr><tr><td>failedDetails</td><td>MixFailedDetail[]</td><td>Detailed information about failed updates</td></tr></table> |
|
|
233
|
+
|
|
234
|
+
`MixFailedDetail`
|
|
235
|
+
|
|
236
|
+
| Name | Type | Description |
|
|
237
|
+
| ----- | -------- | ------------------- |
|
|
238
|
+
| id | string | Id of the failed input source |
|
|
239
|
+
| error | Error | Reason for the failure |
|
|
240
|
+
|
|
241
|
+
### trtc.updatePlugin('VideoMixer', options)
|
|
242
|
+
|
|
243
|
+
Update mixing input sources and layout. The options structure is the same as startPlugin, canvasInfo is not required when updating.
|
|
244
|
+
|
|
245
|
+
### trtc.stopPlugin('VideoMixer')
|
|
246
|
+
|
|
247
|
+
Stop mixing.
|
|
248
|
+
|
|
249
|
+
## Core Parameter Structures
|
|
250
|
+
|
|
251
|
+
### <a name="canvasinfo"></a>CanvasInfo
|
|
252
|
+
|
|
253
|
+
| Name | Type | Required | Description |
|
|
254
|
+
| ----------- | --------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------- |
|
|
255
|
+
| canvasColor | string \| [CanvasGradient](https://developer.mozilla.org/en-US/docs/Web/API/CanvasGradient) \| [CanvasPattern](https://developer.mozilla.org/en-US/docs/Web/API/CanvasPattern) | Optional | Mixing background color, refer to [canvas fillStyle](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/fillStyle) attribute |
|
|
256
|
+
| width | number | Required | Mixing canvas width |
|
|
257
|
+
| height | number | Required | Mixing canvas height |
|
|
258
|
+
| frameRate | number | Optional | Manually specify mixing frame rate. Generally it's best to let the mixing internally calculate the frame rate automatically, manually specifying may result in performance loss |
|
|
259
|
+
|
|
260
|
+
### <a name="camerasource"></a>CameraSource
|
|
261
|
+
|
|
262
|
+
| Name | Type | Required | Description |
|
|
263
|
+
| ---------- | -------------------------------------------------------------------------------------------- | -------- | ----------------------------------- |
|
|
264
|
+
| id | string | Required | Unique id identifying the input source |
|
|
265
|
+
| layout | [LayerOption](#layeroption) | Required | Layout parameters |
|
|
266
|
+
| cameraId | string | Optional | Camera id for capture, specify which camera to capture |
|
|
267
|
+
| videoTrack | MediaStreamTrack | Optional | Custom captured videoTrack |
|
|
268
|
+
| profile | [VideoProfile](https://web.sdk.qcloud.com/trtc/webrtc/v5/doc/en/global.html#VideoProfile) | Optional | Video capture parameters. Default: `480p_2`. |
|
|
269
|
+
|
|
270
|
+
### <a name="screensource"></a>ScreenSource
|
|
271
|
+
|
|
272
|
+
| Name | Type | Required | Description |
|
|
273
|
+
| -------------------- | -------------------------------------------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
274
|
+
| id | string | Required | Unique id identifying the input source |
|
|
275
|
+
| layout | [LayerOption](#layeroption) | Required | Layout parameters |
|
|
276
|
+
| profile | [ScreenShareProfile](https://web.sdk.qcloud.com/trtc/webrtc/v5/doc/en/global.html#ScreenShareProfile) | Optional | Screen capture parameters. Default: `1080p`. |
|
|
277
|
+
| captureElement | HTMLElement | Optional | Capture a specific HTMLElement on the current page, supports Chrome 104+. |
|
|
278
|
+
| preferDisplaySurface | 'current-tab' \| 'tab' \| 'window' \| 'monitor' | Optional | Controls the display priority of different types of capture in the screen sharing selection box. Default is monitor, i.e., screen capture is prioritized in the screen sharing selection box. If set to 'current-tab', the selection box will only show the current page. Supports Chrome 94+. |
|
|
279
|
+
| systemAudio | boolean | Optional | Whether to capture system audio is the same as the systemAudio function for screen capture; it is not captured by default. Supported in v5.15 and later. |
|
|
280
|
+
|
|
281
|
+
### <a name="textsource"></a>TextSource
|
|
282
|
+
|
|
283
|
+
| Name | Type | Required | Description |
|
|
284
|
+
| ------- | --------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------- |
|
|
285
|
+
| id | string | Required | Unique id identifying the input source |
|
|
286
|
+
| layout | [LayerOption](#layeroption) | Required | Layout parameters |
|
|
287
|
+
| content | string | Required | Text content |
|
|
288
|
+
| color | string \| [CanvasGradient](https://developer.mozilla.org/en-US/docs/Web/API/CanvasGradient) \| [CanvasPattern](https://developer.mozilla.org/en-US/docs/Web/API/CanvasPattern)| Optional | Text color, refer to [canvas fillStyle](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/fillStyle) attribute |
|
|
289
|
+
| font | string | Optional | Font, refer to [canvas font](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/font) attribute |
|
|
290
|
+
|
|
291
|
+
### <a name="imagesource"></a>ImageSource
|
|
292
|
+
|
|
293
|
+
| Name | Type | Required | Description |
|
|
294
|
+
| ------ | --------------------------- | -------- | ------------------- |
|
|
295
|
+
| id | string | Required | Unique id identifying the input source |
|
|
296
|
+
| layout | [LayerOption](#layeroption) | Required | Layout parameters |
|
|
297
|
+
| url | string | Required | Image url |
|
|
298
|
+
|
|
299
|
+
### <a name="videosource"></a>VideoSource
|
|
300
|
+
|
|
301
|
+
| Name | Type | Required | Description |
|
|
302
|
+
| ------ | --------------------------- | -------- | ------------------- |
|
|
303
|
+
| id | string | Required | Unique id identifying the input source |
|
|
304
|
+
| layout | [LayerOption](#layeroption) | Required | Layout parameters |
|
|
305
|
+
| url | string | Required | Video url |
|
|
306
|
+
|
|
307
|
+
### <a name="layeroption"></a>LayerOption
|
|
308
|
+
|
|
309
|
+
| Name | Type | Required | Description |
|
|
310
|
+
| -------- | ---------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
311
|
+
| x | number | Required | Horizontal coordinate relative to the upper left corner of the canvas, can be negative |
|
|
312
|
+
| y | number | Required | Vertical coordinate relative to the upper left corner of the canvas, can be negative |
|
|
313
|
+
| width | number | Required | Layout width of input source, must be > 0 |
|
|
314
|
+
| height | number | Required | Layout height of input source, must be > 0 |
|
|
315
|
+
| zIndex | number | Required | Input source layer level, each input source's layer level should not be duplicated |
|
|
316
|
+
| fillMode | 'contain' \| 'cover' \| 'fill' | Optional | Mode for filling the layout with the image, camera defaults to `cover`, other input sources default to `contain`, refer to [CSS object-fit](https://developer.mozilla.org/en-US/docs/Web/CSS/object-fit) attribute. |
|
|
317
|
+
| mirror | boolean | Optional | Whether to mirror |
|
|
318
|
+
| rotation | 0 \| 90 \| 180 \| 270 | Optional | Image rotation angle |
|
|
319
|
+
| hidden | boolean | Optional | Whether to temporarily hide the input source on the canvas. Use case: don't want to physically close camera/screen, temporarily hide input source without destroying resources |
|
|
320
|
+
|
|
321
|
+
## Best Practices and FAQs
|
|
322
|
+
|
|
323
|
+
### 1. Frame Rate and Performance Optimization
|
|
324
|
+
|
|
325
|
+
* **Frame Rate Settings:**
|
|
326
|
+
* **Not Recommended**: manually setting canvasInfo.frameRate.
|
|
327
|
+
* **Recommended**: let the plugin automatically calculate internally: when camera or screen sharing exists, take the maximum frame rate of the two; otherwise fixed at 15 fps. Manual specification may cause performance loss.
|
|
328
|
+
* **Performance Considerations:**
|
|
329
|
+
* The larger the canvas resolution and the more mixing input sources, the greater the rendering overhead. The business side should determine appropriate canvas parameters and number of input sources based on user device performance.
|
|
330
|
+
|
|
331
|
+
### 2. How to Set Resolution and Bitrate?
|
|
332
|
+
|
|
333
|
+
- Resolution is the canvas width and height
|
|
334
|
+
- Bitrate is not controlled by the mixing plugin, it's set by the business side when publishing through `startLocalVideo/updateLocalVideo`.
|
|
335
|
+
|
|
336
|
+
### 3. Why Are Input Source Parameters in Array Form? How to Update Only One Type of Input Source in the Array?
|
|
337
|
+
|
|
338
|
+
- Each type of input source is updated in full. The purpose is to support batch updates and deletions (by comparing arrays passed before and after).
|
|
339
|
+
- Each input source parameter is optional. For example, if you only want to update camera, only pass the camera array when calling updatePlugin, and don't pass arrays for other types of input sources, or pass them together but keep the original parameters. Example:
|
|
340
|
+
|
|
341
|
+
```js
|
|
342
|
+
await trtc.startPlugin('VideoMixer', { // Assume camera and screen are already mixed
|
|
343
|
+
camera: [/* config */],
|
|
344
|
+
screen: [/* config */]
|
|
345
|
+
});
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
At this point, if you only want to update camera and not screen, you can pass only camera without passing screen
|
|
349
|
+
|
|
350
|
+
```js
|
|
351
|
+
await trtc.updatePlugin('VideoMixer', {
|
|
352
|
+
camera: [/* newConfig */], // Only pass camera array
|
|
353
|
+
});
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
### 4. Given an Array, How to Update Only One Input Source?
|
|
357
|
+
|
|
358
|
+
For example, to update a camera with id = 1, only modify the parameters of the element with id = 1 in the camera array passed in. Although other elements don't need to be updated, their parameters need to be passed in as is.
|
|
359
|
+
|
|
360
|
+
### 5. Why Is the Bitrate Set When Capturing Camera and Screen Invalid?
|
|
361
|
+
|
|
362
|
+
The bitrate set when capturing camera and screen will be ignored. The uplink bitrate is set by the interface (`start/updateLocalVideo`) called when the business side publishes.
|
|
363
|
+
|
|
364
|
+
### 6. Why Doesn't Rotation Happen Around the Center?
|
|
365
|
+
|
|
366
|
+
The principle of mixing rotation is to rotate the image and then draw it while keeping the upper left corner of the image at the coordinates passed in by layout. If you want to achieve center rotation, the business side needs to adjust the coordinates accordingly.
|
|
367
|
+
|
|
368
|
+
### 7. If User Stops Screen Sharing via Browser Button Instead of SDK During Mixing Screen Sharing, How Can the Business Side Detect This?
|
|
369
|
+
|
|
370
|
+
The `onScreenShareStop` callback is provided in the parameters of startPlugin. Usage example:
|
|
371
|
+
|
|
372
|
+
```js
|
|
373
|
+
await trtc.startPlugin('VideoMixer', {
|
|
374
|
+
// ...
|
|
375
|
+
onScreenShareStop: (id: string) => {
|
|
376
|
+
console.log(`screen share stop, id: ${id}`);
|
|
377
|
+
}
|
|
378
|
+
});
|
|
379
|
+
```
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
## Function Introduction
|
|
2
|
+
|
|
3
|
+
This article mainly introduces how to use the SmallStreamAutoSwitcher plugin to enable automatic switching between large and small streams. This plugin is typically used in bandwidth-limited scenarios, where if bandwidth cannot meet the requirements for large streams causing stuttering, it automatically switches to small streams, which reduces resolution but ensures smoothness.
|
|
4
|
+
|
|
5
|
+
## Prerequisites
|
|
6
|
+
|
|
7
|
+
- TRTC Web SDK version >= 5.11.0
|
|
8
|
+
|
|
9
|
+
## Implementation Process
|
|
10
|
+
|
|
11
|
+
### Import and Register Plugin
|
|
12
|
+
|
|
13
|
+
```javascript
|
|
14
|
+
import SmallStreamAutoSwitcher from 'trtc-sdk-v5/plugins/small-stream-auto-switcher';
|
|
15
|
+
const trtc = TRTC.create({ plugins: [SmallStreamAutoSwitcher] });
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Plugin Start
|
|
19
|
+
|
|
20
|
+
The default switching strategy is:
|
|
21
|
+
- When network delay exceeds 200ms or packet loss rate exceeds 20% for 10 consecutive seconds, and frame rate fluctuates significantly, switch to small stream
|
|
22
|
+
- When network delay is below 100ms and packet loss rate is below 10% for 20 consecutive seconds, and frame rate returns to normal, switch back to large stream
|
|
23
|
+
- Only allow automatic switching to small stream 2 times, after exceeding 2 times, no longer automatically switch to small stream
|
|
24
|
+
|
|
25
|
+
```javascript
|
|
26
|
+
trtc.startPlugin('SmallStreamAutoSwitcher');
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Plugin Stop
|
|
30
|
+
|
|
31
|
+
```javascript
|
|
32
|
+
trtc.stopPlugin('SmallStreamAutoSwitcher');
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Advanced Usage
|
|
36
|
+
|
|
37
|
+
You can customize parameters to adjust the plugin's strategy, for example:
|
|
38
|
+
|
|
39
|
+
Only allow automatic switching to small stream 1 time, and do not switch back to large stream after switching.
|
|
40
|
+
|
|
41
|
+
```javascript
|
|
42
|
+
trtc.startPlugin('SmallStreamAutoSwitcher', {
|
|
43
|
+
maxAutoSwitchToSmallCount: 1,
|
|
44
|
+
});
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Below are explanations for all parameters. If not passed, default strategy will be used.
|
|
48
|
+
|
|
49
|
+
```javascript
|
|
50
|
+
trtc.startPlugin('SmallStreamAutoSwitcher', {
|
|
51
|
+
rttPoorLimit: 200, // Delay threshold, unit: ms, when delay exceeds this value, triggers condition to switch to small stream, default 200
|
|
52
|
+
lossPoorLimit: 20, // Packet loss rate threshold, unit: %, when packet loss rate exceeds this value, triggers condition to switch to small stream, default 20
|
|
53
|
+
poorDuration: 10 * 1000, // Duration threshold for poor network conditions, unit: ms, when delay or packet loss rate continuously exceeds threshold for this duration, start checking whether to switch to small stream, default 10000 (10 seconds)
|
|
54
|
+
rttGoodLimit: 100, // Good delay threshold, unit: ms, when delay is below this value, triggers condition to switch to large stream, default 100
|
|
55
|
+
lossGoodLimit: 10, // Good packet loss rate threshold, unit: %, when packet loss rate is below this value, triggers condition to switch to large stream, default 10
|
|
56
|
+
goodDuration: 20 * 1000, // Duration threshold for good network conditions, unit: ms, when delay and packet loss rate are continuously below threshold for this duration, start checking whether to switch to large stream, default 20000 (20 seconds)
|
|
57
|
+
sleepTime: 30 * 1000, // Sleep time after switching, unit: ms, no automatic switching will occur during this time, default 30000 (30 seconds)
|
|
58
|
+
maxAutoSwitchToSmallCount: 2, // Maximum number of automatic switches to small stream per user, after exceeding this count, no longer automatically switch to small stream, default 2
|
|
59
|
+
});
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Parameter Description
|
|
63
|
+
|
|
64
|
+
| Parameter Name | Type | Default Value | Description |
|
|
65
|
+
|--------|------|--------|------|
|
|
66
|
+
| rttPoorLimit | number | 200 | Delay threshold (ms), when network delay exceeds this value, start counting conditions for switching to small stream |
|
|
67
|
+
| lossPoorLimit | number | 20 | Packet loss rate threshold (%), when packet loss rate exceeds this value, start counting conditions for switching to small stream |
|
|
68
|
+
| poorDuration | number | 10000 | Duration threshold for poor network conditions (ms), when delay or packet loss rate continuously exceeds threshold for this duration, start checking whether to switch to small stream, default 10000 (10 seconds) |
|
|
69
|
+
| rttGoodLimit | number | 100 | Good delay threshold (ms), when network delay is below this value, start counting conditions for switching to large stream |
|
|
70
|
+
| lossGoodLimit | number | 10 | Good packet loss rate threshold (%), when packet loss rate is below this value, start counting conditions for switching to large stream |
|
|
71
|
+
| goodDuration | number | 20000 | Duration threshold for good network conditions (ms), when delay and packet loss rate are continuously below threshold for this duration, start checking whether to switch to large stream, default 20000 (20 seconds) |
|
|
72
|
+
| sleepTime | number | 30000 | Sleep time after switching (ms), no automatic switching will be triggered again during this period to avoid frequent switching |
|
|
73
|
+
| maxAutoSwitchToSmallCount | number | 2 | Maximum number of automatic switches to small stream allowed per user, after exceeding this count, no longer automatically switch to small stream |
|
|
74
|
+
|
|
75
|
+
## Switching Logic Description
|
|
76
|
+
|
|
77
|
+
### Conditions for Switching to Small Stream
|
|
78
|
+
1. Network conditions: Delay continuously exceeds `rttPoorLimit` for `poorCount` times, or packet loss rate continuously exceeds `lossPoorLimit` for `poorCount` times
|
|
79
|
+
2. Frame rate conditions: Large stream frame rate fluctuates significantly (difference between maximum and minimum frame rate exceeds 1/3 of maximum frame rate)
|
|
80
|
+
3. Limiting conditions: User is currently subscribed to large stream, automatic switch count has not reached `maxAutoSwitchToSmallCount` limit, and not in sleep period
|
|
81
|
+
|
|
82
|
+
### Conditions for Switching to Large Stream
|
|
83
|
+
1. Network conditions: Delay is continuously below `rttGoodLimit` for `goodCount` times, and packet loss rate is continuously below `lossGoodLimit` for `goodCount` times
|
|
84
|
+
2. Frame rate conditions: Small stream frame rate is stable (difference between maximum and minimum frame rate does not exceed 1/3 of maximum frame rate)
|
|
85
|
+
3. Limiting conditions: User is currently subscribed to small stream, and the user has only switched to small stream once, and not in sleep period
|
|
86
|
+
|
|
87
|
+
> **Note:**
|
|
88
|
+
> - Detection frequency is once every 2 seconds
|
|
89
|
+
> - The statistical window size for network quality and frame rate data automatically adapts to the maximum value of poorDuration and goodDuration
|
|
90
|
+
> - After each switch, it enters a sleep period of `sleepTime` duration, during which automatic switching will not be triggered again
|
|
91
|
+
> - Only users who have automatically switched to small stream once are allowed to switch back to large stream, users who have switched twice will remain in small stream state
|