@xmaxai/web-sdk 1.0.10 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -64
- package/dist/{TrajectoryEffectRendering-BKKPO7Uk.d.cts → TrajectoryEffectRendering-BdSJxHQw.d.cts} +24 -33
- package/dist/{TrajectoryEffectRendering-BKKPO7Uk.d.ts → TrajectoryEffectRendering-BdSJxHQw.d.ts} +24 -33
- package/dist/chunk-IJVTTKBG.js +14591 -0
- package/dist/chunk-IJVTTKBG.js.map +1 -0
- package/dist/chunk-WJOPIHWW.cjs +14591 -0
- package/dist/chunk-WJOPIHWW.cjs.map +1 -0
- package/dist/index.cjs +10 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +899 -325
- package/dist/index.d.ts +899 -325
- package/dist/index.js +19 -12
- package/dist/react/index.cjs +15 -10
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +16 -6
- package/dist/react/index.d.ts +16 -6
- package/dist/react/index.js +18 -13
- package/dist/react/index.js.map +1 -1
- package/package.json +9 -10
- package/dist/RecordingProcessor-4TP5NICO.cjs +0 -211
- package/dist/RecordingProcessor-4TP5NICO.cjs.map +0 -1
- package/dist/RecordingProcessor-ZSBJFF4J.js +0 -211
- package/dist/RecordingProcessor-ZSBJFF4J.js.map +0 -1
- package/dist/chunk-4DE2IREA.cjs +0 -8
- package/dist/chunk-4DE2IREA.cjs.map +0 -1
- package/dist/chunk-LOYJFYQT.js +0 -14968
- package/dist/chunk-LOYJFYQT.js.map +0 -1
- package/dist/chunk-UF2RD42G.cjs +0 -14968
- package/dist/chunk-UF2RD42G.cjs.map +0 -1
- package/dist/chunk-V6TY7KAL.js +0 -8
- package/dist/chunk-V6TY7KAL.js.map +0 -1
package/README.md
CHANGED
|
@@ -217,14 +217,14 @@ after a user action such as clicking a Start button.
|
|
|
217
217
|
```javascript
|
|
218
218
|
import {
|
|
219
219
|
CameraPosition,
|
|
220
|
+
RealtimeCameraPreset,
|
|
220
221
|
RealtimeConfiguration,
|
|
221
222
|
RealtimeContext,
|
|
222
223
|
RealtimeModel,
|
|
223
|
-
RealtimeVideoFormat,
|
|
224
224
|
VideoContentMode,
|
|
225
225
|
XmaxClient,
|
|
226
226
|
XmaxConfiguration,
|
|
227
|
-
|
|
227
|
+
XmaxRealtimeDualVideoView,
|
|
228
228
|
} from "@xmaxai/web-sdk";
|
|
229
229
|
|
|
230
230
|
const client = new XmaxClient(
|
|
@@ -232,17 +232,17 @@ const client = new XmaxClient(
|
|
|
232
232
|
);
|
|
233
233
|
|
|
234
234
|
const realtime = client.createRealtimeManager(
|
|
235
|
-
new RealtimeConfiguration({ model: RealtimeModel.
|
|
235
|
+
new RealtimeConfiguration({ model: RealtimeModel.x2_1 })
|
|
236
236
|
);
|
|
237
237
|
|
|
238
238
|
const localStream = await realtime.createLocalCameraStream({
|
|
239
|
-
|
|
239
|
+
preset: RealtimeCameraPreset.resolution1080x1920,
|
|
240
240
|
position: CameraPosition.front,
|
|
241
241
|
useMicrophone: false,
|
|
242
242
|
enableFrameValidation: true, // Optional; waits a fixed 200ms camera warmup before publishing; false skips it.
|
|
243
243
|
});
|
|
244
244
|
|
|
245
|
-
const videoView = new
|
|
245
|
+
const videoView = new XmaxRealtimeDualVideoView({
|
|
246
246
|
localTrack: localStream.videoTrack,
|
|
247
247
|
videoContentMode: VideoContentMode.fill,
|
|
248
248
|
});
|
|
@@ -257,7 +257,7 @@ const remoteStream = await realtime.startGeneration({
|
|
|
257
257
|
localStream,
|
|
258
258
|
context: new RealtimeContext({
|
|
259
259
|
prompt: "视频中角色替换成参考图中角色",
|
|
260
|
-
referencePath: "https://
|
|
260
|
+
referencePath: "https://cdn.22duck.cn/images/source/charx/chatx_image1.jpg",
|
|
261
261
|
}),
|
|
262
262
|
});
|
|
263
263
|
|
|
@@ -267,8 +267,47 @@ videoView.remoteTrack = remoteStream.videoTrack;
|
|
|
267
267
|
Add a `video-container` element with an explicit width and height to your page.
|
|
268
268
|
The view displays a local camera preview until the first generated frame arrives.
|
|
269
269
|
|
|
270
|
+
Provide either `preset` or `videoFormat`, not both. A preset selects a model-supported
|
|
271
|
+
resolution and the model's recommended frame rate. To specify your own frame rate
|
|
272
|
+
or bitrate options, pass a `RealtimeVideoFormat` through `videoFormat` instead.
|
|
273
|
+
Models without fixed presets, such as X2.0, require `videoFormat`.
|
|
274
|
+
|
|
275
|
+
`RealtimeModel.x2_1` uses Agora and supports 1920×1080, 1080×1920, 1408×1408,
|
|
276
|
+
1152×1536, and 1536×1152. `RealtimeModel.x2_0` uses VeRTC, defaults to 832×1472,
|
|
277
|
+
and resolves other dimensions within its pixel budget. Known models use the realtime
|
|
278
|
+
endpoint for `XmaxEnvironment.china` or `XmaxEnvironment.global`; custom model names
|
|
279
|
+
are sent unchanged to the test endpoint.
|
|
280
|
+
|
|
281
|
+
Once the camera preview reaches `RealtimeConnectionState.ready`, you can change
|
|
282
|
+
its format before connecting. Presets retain the current frame rate and bitrate options:
|
|
283
|
+
|
|
284
|
+
```javascript
|
|
285
|
+
import { RealtimeCameraPreset } from "@xmaxai/web-sdk";
|
|
286
|
+
|
|
287
|
+
await realtime.updateCameraFormat(RealtimeCameraPreset.resolution1408x1408);
|
|
288
|
+
console.log(realtime.currentCameraVideoFormat, realtime.currentCameraPreset);
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
During generation, select an uplink or downlink quality level directly:
|
|
292
|
+
|
|
293
|
+
```javascript
|
|
294
|
+
import { RealtimeQualityLevel } from "@xmaxai/web-sdk";
|
|
295
|
+
|
|
296
|
+
await realtime.setUplinkQualityLevel(RealtimeQualityLevel.l3);
|
|
297
|
+
await realtime.setDownlinkQualityLevel(RealtimeQualityLevel.l2);
|
|
298
|
+
console.log(realtime.uplinkQualityLevel, realtime.downlinkQualityLevel);
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Automatic adaptation continues after a manual change on supported RTC providers;
|
|
302
|
+
VeRTC applies manual levels only. L5 scales width and height to 50% of the media
|
|
303
|
+
baseline. The default uplink minimum bitrate is 500 kbps, capped at the selected
|
|
304
|
+
maximum bitrate; explicit bitrate settings take precedence.
|
|
305
|
+
|
|
270
306
|
Use `updateUplinkVideoFormat` after creating the local camera stream, including during
|
|
271
|
-
generation, to change upstream encoding without reconnecting or restarting the task
|
|
307
|
+
generation, to change upstream encoding without reconnecting or restarting the task.
|
|
308
|
+
|
|
309
|
+
Dimensions are rounded down to multiples of four, preserving the exact aspect ratio
|
|
310
|
+
when possible. The local track's `videoFormat` reflects the aligned dimensions.
|
|
272
311
|
|
|
273
312
|
```javascript
|
|
274
313
|
await realtime.updateUplinkVideoFormat(new RealtimeVideoFormat({
|
|
@@ -285,9 +324,9 @@ await realtime.updateUplinkVideoFormat(new RealtimeVideoFormat({
|
|
|
285
324
|
### Generate from a local image
|
|
286
325
|
|
|
287
326
|
Use `createLocalImageStream` instead of `createLocalCameraStream` to use a local
|
|
288
|
-
image as the video input. It accepts a browser-decodable `File` or `Blob`,
|
|
289
|
-
|
|
290
|
-
|
|
327
|
+
image as the video input. It accepts a browser-decodable `File` or `Blob`, preserves
|
|
328
|
+
the complete image, and continuously sends the same frame without audio.
|
|
329
|
+
No image upload or camera permission is required.
|
|
291
330
|
|
|
292
331
|
```javascript
|
|
293
332
|
const localStream = await realtime.createLocalImageStream({
|
|
@@ -306,74 +345,163 @@ TRTC, Agora, and VeRTC use the existing external-video publishing path.
|
|
|
306
345
|
`disconnect()` retains the image for reconnection; `close()` releases its resources.
|
|
307
346
|
Animated images are captured as a single still frame.
|
|
308
347
|
|
|
348
|
+
For local images and local videos, the SDK reads the actual media dimensions,
|
|
349
|
+
limits the pixel area, and aligns the result to multiples of 4 without enlarging it.
|
|
350
|
+
The `videoFormat` width and height remain accepted for compatibility; frame rate
|
|
351
|
+
and encoding settings are retained. External video tracks use the same size rules;
|
|
352
|
+
pass their original dimensions in `createExternalVideoStream({ videoTrack, videoFormat })`.
|
|
353
|
+
The SDK selects a separate model generation canvas without processing video frames.
|
|
354
|
+
It sends `padding: "contain"` only when the resolved media and canvas aspect ratios
|
|
355
|
+
differ. Requested output dimensions follow the fixed media size and downlink quality
|
|
356
|
+
tier. Uplink quality changes do not change the generation canvas or media baseline.
|
|
357
|
+
|
|
358
|
+
Camera, local-image, local-video, and external-track gestures use normalized
|
|
359
|
+
coordinates in `[0, 1]` relative to their media, independent of encoding quality changes.
|
|
360
|
+
|
|
361
|
+
### Supply externally captured video and PCM
|
|
362
|
+
|
|
363
|
+
Use `createExternalVideoStream({ source })` to bind a `RealtimeExternalVideoSource`.
|
|
364
|
+
The source declares its original `videoFormat` and optional `hasAudio` (default
|
|
365
|
+
`false`), installs capture/decoder callbacks in `start(sink)`, and removes them in
|
|
366
|
+
`stop()`. Your app continues to own its devices, decoder, and original frames.
|
|
367
|
+
|
|
368
|
+
```typescript
|
|
369
|
+
import type { RealtimeExternalFrameSink, RealtimeExternalVideoSource } from "@xmaxai/web-sdk";
|
|
370
|
+
import { RealtimeVideoFormat } from "@xmaxai/web-sdk";
|
|
371
|
+
|
|
372
|
+
let input: RealtimeExternalFrameSink | undefined;
|
|
373
|
+
const source: RealtimeExternalVideoSource = {
|
|
374
|
+
videoFormat: new RealtimeVideoFormat({ width: 1280, height: 720, fps: 30 }),
|
|
375
|
+
hasAudio: true,
|
|
376
|
+
async start(sink) {
|
|
377
|
+
input = sink;
|
|
378
|
+
// Install your capture/decoder callbacks here. Do not wait for generation.
|
|
379
|
+
},
|
|
380
|
+
async stop() {
|
|
381
|
+
input = undefined;
|
|
382
|
+
// Remove callbacks here. Keep this safe to call while startup is pending.
|
|
383
|
+
},
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
// Call from a user gesture when supplying audio.
|
|
387
|
+
const local = await realtime.createExternalVideoStream({ source });
|
|
388
|
+
// Bind local.videoTrack to an SDK video view, then connect/startGeneration as usual.
|
|
389
|
+
|
|
390
|
+
function onCapturedVideo(frame: VideoFrame) {
|
|
391
|
+
try { input?.pushVideo(frame); } finally { frame.close(); }
|
|
392
|
+
}
|
|
393
|
+
function onDecodedAudio(frame: AudioData) {
|
|
394
|
+
try { input?.pushAudio(frame); } finally { frame.close(); }
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Once disconnected, detach the source and invalidate its old sink:
|
|
398
|
+
await realtime.disconnect();
|
|
399
|
+
await realtime.stopExternalVideoStream();
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
- Both timestamps are nonnegative microseconds on one monotonic source timeline.
|
|
403
|
+
Keep timestamps increasing across loops and seeks; pace submission in real time.
|
|
404
|
+
Repeated/backward video times and overlapping audio buffers are ignored.
|
|
405
|
+
- Submit video at the declared source dimensions, with rotation/mirroring already
|
|
406
|
+
applied. The SDK scales to the resolved media size and provides local preview;
|
|
407
|
+
camera resolution buckets do not apply. `sink.videoFormat` exposes this fixed size.
|
|
408
|
+
- Audio must be decoded PCM (`AudioData`), 8–192 kHz, 1–8 channels, at most one second
|
|
409
|
+
per packet. Its format must remain fixed during a binding. Samples are copied,
|
|
410
|
+
downmixed to mono and resampled through AudioWorklet (some browsers expose a
|
|
411
|
+
two-channel track containing identical mono samples). Browser audio tracks are
|
|
412
|
+
continuous, so gaps become silence; original PTS cannot be forwarded as native
|
|
413
|
+
RTC packet timestamps. There is a shared 20 ms scheduling allowance and bounded
|
|
414
|
+
queuing, not an exact replica of native RTC frame submission.
|
|
415
|
+
- Audio is uploaded only after RTC publication. `disconnect()` flushes queued
|
|
416
|
+
audio but preserves the source/video preview; reconnection does not replay old
|
|
417
|
+
audio. No microphone permission is requested and no input audio is played locally.
|
|
418
|
+
- `source.start()` must return when callbacks are installed, without waiting for a
|
|
419
|
+
first frame or generation. Startup is limited to 10 seconds; `stop()` is called
|
|
420
|
+
after startup failure/cancellation too. `stopExternalVideoStream()` and `close()`
|
|
421
|
+
invalidate the sink, clear preview, and release SDK-created tracks. A source error
|
|
422
|
+
reported via `sink.reportError(error)` ends the connection and local stream.
|
|
423
|
+
- Video input requires `VideoFrame` and Canvas capture. Audio additionally requires
|
|
424
|
+
a secure context, AudioWorklet and an unlocked AudioContext. This path uses the
|
|
425
|
+
existing TRTC, Agora and VeRTC external-track adapters; it does not upload files.
|
|
426
|
+
|
|
427
|
+
The existing `createExternalVideoStream({ videoTrack, videoFormat })` form remains
|
|
428
|
+
video-only and keeps its clone/ownership behavior. Do not combine it with `source`.
|
|
429
|
+
|
|
309
430
|
<br>
|
|
310
431
|
|
|
311
|
-
###
|
|
432
|
+
### Observe remote audio and video frames
|
|
312
433
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
434
|
+
The SDK does not record or package files. Your app decides when to observe media,
|
|
435
|
+
encode it, and stop recording. Register from a user gesture (for example a button
|
|
436
|
+
click) and await setup before expecting callbacks:
|
|
316
437
|
|
|
317
438
|
```javascript
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
439
|
+
await realtime.setRemoteMediaFrameListener(packet => {
|
|
440
|
+
if (packet.content.type === "video") {
|
|
441
|
+
const { frame } = packet.content.frame;
|
|
442
|
+
try {
|
|
443
|
+
// Submit synchronously to your encoder, or transfer ownership to your worker.
|
|
444
|
+
// Both tracks use packet.presentationTimeStamp in microseconds.
|
|
445
|
+
console.log(packet.streamID, packet.presentationTimeStamp, frame.displayWidth);
|
|
446
|
+
} finally {
|
|
447
|
+
frame.close(); // If retained/transferred asynchronously, release it there instead.
|
|
448
|
+
}
|
|
449
|
+
} else {
|
|
450
|
+
const { data, sampleRate, channels, samplesPerChannel } = packet.content.frame;
|
|
451
|
+
// Owned S16LE, interleaved PCM; data remains valid after this callback.
|
|
452
|
+
console.log(data, sampleRate, channels, samplesPerChannel);
|
|
324
453
|
}
|
|
325
|
-
|
|
326
|
-
const { blob, hasAudio, reason } = event.result;
|
|
327
|
-
// Save, upload, or preview blob here. Use blob.type to identify its format.
|
|
328
|
-
// completed: reached the time limit; interrupted: finalized an earlier end.
|
|
329
|
-
console.log(event.taskID, blob, hasAudio, reason);
|
|
330
454
|
});
|
|
331
455
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
context: new RealtimeContext({ prompt: "Make the scene come alive" }),
|
|
335
|
-
record: true,
|
|
336
|
-
});
|
|
456
|
+
// Stop observing at any time, without stopping generation or changing playback:
|
|
457
|
+
await realtime.setRemoteMediaFrameListener(undefined);
|
|
337
458
|
```
|
|
338
459
|
|
|
339
|
-
-
|
|
340
|
-
|
|
341
|
-
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
460
|
+
- Video is copied from the original remote track, excluding
|
|
461
|
+
mirroring, cropping, interaction effects, and UI overlays.
|
|
462
|
+
- Audio is taken before playback-volume processing. Muting playback does not mute
|
|
463
|
+
observed PCM on supported platforms. No synthetic silence is added when no audio
|
|
464
|
+
track exists; audio published later is observed after subscription.
|
|
465
|
+
- Audio and video share a receiver presentation clock, not sender/source-file
|
|
466
|
+
timecode. Use one recording origin for both tracks; do not pair packets by
|
|
467
|
+
callback arrival time. Audio time advances by sample count between discontinuities.
|
|
468
|
+
- A new result binding has a new opaque streamID. The first packet of each track,
|
|
469
|
+
format/clock changes, and dropped output are marked isDiscontinuous. Your app
|
|
470
|
+
decides whether to retain gaps, segment, or trim both tracks. The SDK neither
|
|
471
|
+
compresses timestamps nor imposes a recording-duration limit.
|
|
472
|
+
- Callbacks run on the main thread in delivery order, not cross-track PTS order.
|
|
473
|
+
Keep them lightweight and move encoding to a worker. Pending SDK delivery is
|
|
474
|
+
bounded to 256 packets and 32 MiB; already-delivered video frames belong to your
|
|
475
|
+
app and must be closed. Removing/replacing the listener discards queued output.
|
|
476
|
+
- Requires a secure context, VideoFrame, requestVideoFrameCallback, and AudioWorklet.
|
|
477
|
+
Registration rejects when setup is unavailable, cancelled, or blocked by audio
|
|
478
|
+
autoplay policy. TRTC on mobile Safari currently cannot provide PCM independently
|
|
479
|
+
of playback and is rejected without interrupting generation.
|
|
480
|
+
- Browser frame callbacks can skip frames, especially under main-thread load or
|
|
481
|
+
in background tabs. This is not a lossless dump of every decoded frame.
|
|
482
|
+
- Without a listener, no observation video, audio graph, or output queue is started.
|
|
483
|
+
Disconnect preserves the listener for a later connection; close removes it and
|
|
484
|
+
releases the audio context. Re-register before reusing a closed manager.
|
|
364
485
|
|
|
365
486
|
<br>
|
|
366
487
|
|
|
367
488
|
### Using React
|
|
368
489
|
|
|
369
|
-
|
|
490
|
+
`XmaxRealtimeVideoView` displays a single local or remote track;
|
|
491
|
+
`XmaxRealtimeDualVideoView` overlays local and remote tracks and switches between them.
|
|
492
|
+
Their React counterparts are `XmaxRealtimeVideo` and `XmaxRealtimeDualVideo`.
|
|
493
|
+
The dual view exposes `remoteVideoFadeInDuration` in seconds (default `0.3`),
|
|
494
|
+
also available as a React prop. Zero, negative, or nonfinite values disable the
|
|
495
|
+
fade-in animation without skipping rendering warmup. Changes apply to the next fade-in.
|
|
496
|
+
|
|
497
|
+
Use `XmaxRealtimeDualVideo` as your primary React view. Store the local and remote
|
|
370
498
|
tracks in component state, updating them dynamically as streams become available:
|
|
371
499
|
|
|
372
500
|
```jsx
|
|
373
|
-
import {
|
|
501
|
+
import { XmaxRealtimeDualVideo } from "@xmaxai/web-sdk/react";
|
|
374
502
|
import { VideoContentMode } from "@xmaxai/web-sdk";
|
|
375
503
|
|
|
376
|
-
<
|
|
504
|
+
<XmaxRealtimeDualVideo
|
|
377
505
|
localTrack={localTrack}
|
|
378
506
|
remoteTrack={remoteTrack}
|
|
379
507
|
videoContentMode={VideoContentMode.fill}
|
|
@@ -381,12 +509,12 @@ import { VideoContentMode } from "@xmaxai/web-sdk";
|
|
|
381
509
|
/>
|
|
382
510
|
```
|
|
383
511
|
|
|
384
|
-
See the [React component](./packages/xmax-sdk/src/react/
|
|
512
|
+
See the [React component](./packages/xmax-sdk/src/react/XmaxRealtimeDualVideo.tsx) for state binding and the
|
|
385
513
|
[example project](#example-project) for a complete implementation.
|
|
386
514
|
|
|
387
515
|
### Interact with the generated video
|
|
388
516
|
|
|
389
|
-
`
|
|
517
|
+
`XmaxRealtimeVideo` and `XmaxRealtimeDualVideo` support dragging on the remote output with a
|
|
390
518
|
mouse, pen, or multiple touch points. Interaction becomes available after generation
|
|
391
519
|
starts. The SDK maps the displayed coordinates to the model input and sends the
|
|
392
520
|
trajectory automatically; local previews do not send interaction events.
|
|
@@ -399,6 +527,11 @@ custom visuals, or leave it undefined to use the default glowing trail. A render
|
|
|
399
527
|
only controls visuals, not sampling or signaling; use a separate instance per view
|
|
400
528
|
and release its animation resources in `reset()`.
|
|
401
529
|
|
|
530
|
+
`DefaultTrajectoryEffectRenderer` is also exported for shared interaction overlays
|
|
531
|
+
that need the exact SDK visuals. Mount its `element`, feed viewport CSS-pixel
|
|
532
|
+
`TrajectoryPoint` values through `renderBegan` / `renderMoved` / `renderEnded`,
|
|
533
|
+
and call `reset()` before removing it. This visual renderer does not send tracks.
|
|
534
|
+
|
|
402
535
|
The generated response to dragging depends on the model and generation prompt.
|
|
403
536
|
|
|
404
537
|
<br>
|
|
@@ -411,9 +544,24 @@ input stream or starting generation.
|
|
|
411
544
|
| Listener | Purpose |
|
|
412
545
|
| --- | --- |
|
|
413
546
|
| `setStateListener` | Observe pipeline states and termination reasons during real-time generation. |
|
|
414
|
-
| `
|
|
415
|
-
| `
|
|
416
|
-
| `
|
|
547
|
+
| `setTimingListener` | Monitor camera, connection, first-frame, and total startup timings. A summary is logged when render warmup completes and performance logging is enabled. |
|
|
548
|
+
| `setNetworkQualityListener` | Monitor `uplink` and `downlink` quality using `RealtimeNetworkQualityLevel`. |
|
|
549
|
+
| `setUplinkVideoStatisticsListener` | Monitor resolution, capture/encode/send frame rates, bitrate, and packet loss. |
|
|
550
|
+
| `setDownlinkVideoStatisticsListener` | Monitor resolution, decode/RTC-render frame rates, bitrate, packet loss, and estimated RTC end-to-end delay. |
|
|
551
|
+
|
|
552
|
+
Unavailable video metrics are `undefined`, not inferred from another stage's frame
|
|
553
|
+
rate. Both video statistics types use `packetLossPercent` (0–100) and `bitrateKbps`;
|
|
554
|
+
downlink `renderFrameRate` is reported by the RTC provider. Network quality ranges from
|
|
555
|
+
`unknown` to `excellent`, `good`, `poor`, `bad`, `veryBad`, and `down`.
|
|
556
|
+
RTT and playback-buffer diagnostics remain in RTC performance logs rather than
|
|
557
|
+
these public statistics structures. Listeners receive `undefined` when their
|
|
558
|
+
current snapshot is cleared.
|
|
559
|
+
|
|
560
|
+
Automatic uplink/downlink quality adaptation is enabled by default on supported RTC
|
|
561
|
+
providers. Set `isAdaptiveQualityEnabled: false` in `RealtimeConfiguration` to start
|
|
562
|
+
each connection at L1 without reading or writing quality history. Manual changes
|
|
563
|
+
through `setUplinkQualityLevel` and `setDownlinkQualityLevel` remain available,
|
|
564
|
+
and statistics listeners continue to receive updates.
|
|
417
565
|
|
|
418
566
|
For example, monitor state changes and errors:
|
|
419
567
|
|
|
@@ -441,7 +589,7 @@ the 1000 ms render warmup are complete, then becomes
|
|
|
441
589
|
dismiss your loading UI. Without an SDK video view, the state stays `starting`.
|
|
442
590
|
|
|
443
591
|
At generation start, the SDK sends `params.mirror: true` for the front camera
|
|
444
|
-
and `false` for the rear camera
|
|
592
|
+
and `false` for the rear camera and local files. The backend
|
|
445
593
|
mirrors the input; generated results are not automatically mirrored in the
|
|
446
594
|
browser. The front-camera local preview remains mirrored independently.
|
|
447
595
|
This flag is sent only with `start`: switching cameras during an active generation
|
|
@@ -488,6 +636,19 @@ A complete example application featuring React is available in
|
|
|
488
636
|
[`examples/xlab-react`](./examples/xlab-react).
|
|
489
637
|
It demonstrates real-time generation using live camera feeds and reference images.
|
|
490
638
|
|
|
639
|
+
During generation, click **录制** to record the remote output and click it again to
|
|
640
|
+
stop. A preview/save dialog opens when encoding finishes. The example uses
|
|
641
|
+
`setRemoteMediaFrameListener` and encodes in a Worker; the recorder and its
|
|
642
|
+
`mediabunny` dependency belong to XLab, not the SDK. It prefers MP4 (H.264/AAC),
|
|
643
|
+
with WebM (VP8/Opus) as a fallback. Playback muting does not mute recorded audio on
|
|
644
|
+
supported platforms. Recordings exclude UI overlays, and retain
|
|
645
|
+
the receiver timeline rather than trimming network stalls. Ending a session also
|
|
646
|
+
finishes the current recording. To bound memory, XLab automatically finishes near
|
|
647
|
+
128 MiB of encoded media; overloaded encoders report an error instead of queuing
|
|
648
|
+
unbounded raw frames. Closing/reloading the page discards unsaved recordings.
|
|
649
|
+
Native save-location selection depends on browser support; otherwise the save
|
|
650
|
+
button starts a regular download.
|
|
651
|
+
|
|
491
652
|
<br>
|
|
492
653
|
|
|
493
654
|
## Dependencies
|
package/dist/{TrajectoryEffectRendering-BKKPO7Uk.d.cts → TrajectoryEffectRendering-BdSJxHQw.d.cts}
RENAMED
|
@@ -11,6 +11,9 @@ declare enum RealtimeVideoEncoderPreference {
|
|
|
11
11
|
*/
|
|
12
12
|
maintainQuality = "maintainQuality"
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* 视频尺寸、帧率和上传编码选项。
|
|
16
|
+
*/
|
|
14
17
|
interface RealtimeVideoFormatInit {
|
|
15
18
|
/**
|
|
16
19
|
* 视频宽度,单位为像素。
|
|
@@ -25,7 +28,7 @@ interface RealtimeVideoFormatInit {
|
|
|
25
28
|
*/
|
|
26
29
|
fps: number;
|
|
27
30
|
/**
|
|
28
|
-
* 最低上传码率,单位为 kbps
|
|
31
|
+
* 最低上传码率,单位为 kbps;缺省取 500 与最高码率的较小值,0 表示不设下限。
|
|
29
32
|
*/
|
|
30
33
|
minimumBitrate?: number;
|
|
31
34
|
/**
|
|
@@ -41,9 +44,6 @@ interface RealtimeVideoFormatInit {
|
|
|
41
44
|
* 实时视频的尺寸、帧率和上传编码配置。
|
|
42
45
|
*/
|
|
43
46
|
declare class RealtimeVideoFormat {
|
|
44
|
-
/**
|
|
45
|
-
* 视频规格
|
|
46
|
-
*/
|
|
47
47
|
/**
|
|
48
48
|
* 视频宽度,单位为像素。
|
|
49
49
|
*/
|
|
@@ -57,10 +57,7 @@ declare class RealtimeVideoFormat {
|
|
|
57
57
|
*/
|
|
58
58
|
readonly fps: number;
|
|
59
59
|
/**
|
|
60
|
-
*
|
|
61
|
-
*/
|
|
62
|
-
/**
|
|
63
|
-
* 最低上传码率,单位为 kbps;缺省为 100,0 表示不设最低码率;仅支持该参数的 RTC 链路生效。
|
|
60
|
+
* 最低上传码率,单位为 kbps;缺省取 500 与最高码率的较小值,0 表示不设下限;仅支持该参数的 RTC 链路生效。
|
|
64
61
|
*/
|
|
65
62
|
readonly minimumBitrate?: number;
|
|
66
63
|
/**
|
|
@@ -73,13 +70,6 @@ declare class RealtimeVideoFormat {
|
|
|
73
70
|
readonly encoderPreference: RealtimeVideoEncoderPreference;
|
|
74
71
|
/**
|
|
75
72
|
* 创建实时视频格式。
|
|
76
|
-
*
|
|
77
|
-
* @param init.width 视频宽度,单位为像素。
|
|
78
|
-
* @param init.height 视频高度,单位为像素。
|
|
79
|
-
* @param init.fps 视频帧率,必须大于 0。
|
|
80
|
-
* @param init.minimumBitrate 最低上传码率,单位为 kbps;缺省为 100,仅支持该参数的 RTC 链路生效。
|
|
81
|
-
* @param init.maximumBitrate 最高上传码率,单位为 kbps;缺省按最终上传尺寸和帧率计算。
|
|
82
|
-
* @param init.encoderPreference 上传编码策略偏好,默认值为 maintainFramerate。
|
|
83
73
|
*/
|
|
84
74
|
constructor(init: RealtimeVideoFormatInit);
|
|
85
75
|
/**
|
|
@@ -110,6 +100,9 @@ declare enum CameraPosition {
|
|
|
110
100
|
back = "back"
|
|
111
101
|
}
|
|
112
102
|
|
|
103
|
+
/**
|
|
104
|
+
* 视频轨道身份及初始媒体信息。
|
|
105
|
+
*/
|
|
113
106
|
interface RealtimeVideoTrackInit {
|
|
114
107
|
/**
|
|
115
108
|
* 视频轨道标识。
|
|
@@ -131,9 +124,6 @@ interface RealtimeVideoTrackInit {
|
|
|
131
124
|
* 渲染层通过它把画面绑定到 `<video>` 元素。
|
|
132
125
|
*/
|
|
133
126
|
declare class RealtimeVideoTrack {
|
|
134
|
-
/**
|
|
135
|
-
* 轨道信息
|
|
136
|
-
*/
|
|
137
127
|
/**
|
|
138
128
|
* 视频轨道标识。
|
|
139
129
|
*/
|
|
@@ -142,9 +132,6 @@ declare class RealtimeVideoTrack {
|
|
|
142
132
|
* 动态元数据
|
|
143
133
|
*/
|
|
144
134
|
private metadata;
|
|
145
|
-
/**
|
|
146
|
-
* 平台资源
|
|
147
|
-
*/
|
|
148
135
|
/**
|
|
149
136
|
* SDK 内部:轨道对应的 Web 媒体轨。
|
|
150
137
|
* 本地轨道在采集启动后可用;远端轨道在订阅成功后可用。
|
|
@@ -152,10 +139,6 @@ declare class RealtimeVideoTrack {
|
|
|
152
139
|
mediaStreamTrack?: MediaStreamTrack;
|
|
153
140
|
/**
|
|
154
141
|
* 创建实时视频轨道。
|
|
155
|
-
*
|
|
156
|
-
* @param init.id 视频轨道标识。
|
|
157
|
-
* @param init.videoFormat 轨道初始视频格式。
|
|
158
|
-
* @param init.position 本地相机轨道的初始摄像头位置。
|
|
159
142
|
*/
|
|
160
143
|
constructor(init: RealtimeVideoTrackInit);
|
|
161
144
|
/**
|
|
@@ -195,21 +178,17 @@ declare enum VideoContentMode {
|
|
|
195
178
|
*/
|
|
196
179
|
interface VideoRenderStatistics {
|
|
197
180
|
/**
|
|
198
|
-
*
|
|
181
|
+
* 按浏览器 presentedFrames 增量计数的视频呈现来源。
|
|
199
182
|
*/
|
|
200
|
-
readonly source: "video"
|
|
183
|
+
readonly source: "video";
|
|
201
184
|
/**
|
|
202
|
-
*
|
|
185
|
+
* 视频呈现帧率,单位 fps。
|
|
203
186
|
*/
|
|
204
187
|
readonly frameRate: number;
|
|
205
188
|
/**
|
|
206
189
|
* 原帧呈现帧率,单位 fps。
|
|
207
190
|
*/
|
|
208
191
|
readonly originalFrameRate: number;
|
|
209
|
-
/**
|
|
210
|
-
* 插值帧呈现帧率,单位 fps;未开启插帧时为 0。
|
|
211
|
-
*/
|
|
212
|
-
readonly interpolatedFrameRate: number;
|
|
213
192
|
}
|
|
214
193
|
|
|
215
194
|
/**
|
|
@@ -220,15 +199,27 @@ type TrajectoryID = number;
|
|
|
220
199
|
* 供视觉效果使用的触点;location 为视口 CSS 像素,timestamp 为毫秒。
|
|
221
200
|
*/
|
|
222
201
|
interface TrajectoryPoint {
|
|
202
|
+
/**
|
|
203
|
+
* 同一触点在连续采样期间保持不变的轨迹标识。
|
|
204
|
+
*/
|
|
223
205
|
readonly id: TrajectoryID;
|
|
206
|
+
/**
|
|
207
|
+
* 触点在视口中的位置,单位为 CSS 像素。
|
|
208
|
+
*/
|
|
224
209
|
readonly location: Readonly<{
|
|
225
210
|
x: number;
|
|
226
211
|
y: number;
|
|
227
212
|
}>;
|
|
213
|
+
/**
|
|
214
|
+
* 触点在视口中的归一化位置,横纵范围为 0–1。
|
|
215
|
+
*/
|
|
228
216
|
readonly normalizedLocation: Readonly<{
|
|
229
217
|
x: number;
|
|
230
218
|
y: number;
|
|
231
219
|
}>;
|
|
220
|
+
/**
|
|
221
|
+
* 触点采样时的单调时钟时间,单位为毫秒。
|
|
222
|
+
*/
|
|
232
223
|
readonly timestamp: number;
|
|
233
224
|
}
|
|
234
225
|
|
|
@@ -259,4 +250,4 @@ interface TrajectoryEffectRendering {
|
|
|
259
250
|
reset(): void;
|
|
260
251
|
}
|
|
261
252
|
|
|
262
|
-
export { CameraPosition as C, RealtimeVideoFormat as R, type TrajectoryEffectRendering as T, VideoContentMode as V, RealtimeVideoTrack as a, type VideoRenderStatistics as b,
|
|
253
|
+
export { CameraPosition as C, RealtimeVideoFormat as R, type TrajectoryEffectRendering as T, VideoContentMode as V, RealtimeVideoTrack as a, type VideoRenderStatistics as b, type TrajectoryPoint as c, type TrajectoryID as d, RealtimeVideoEncoderPreference as e, type RealtimeVideoFormatInit as f };
|