@streaming-cdn/rtc-web 1.3.14 → 1.3.16

package/README.md

@@ -1,4 +1,4 @@
-# RTC Web SDK 1.3.14
+# RTC Web SDK 1.3.16
 
 The customer package contains compiled ESM/UMD JavaScript and TypeScript declarations. It intentionally excludes implementation source and source maps. Example source remains available under `examples/`.
 
@@ -169,6 +169,75 @@ the room client (`getNativeClient()`); the publisher keeps sending every
 layer. The React Native adapter has the same transport option and the same
 `setRemoteQuality` on the client.
 
+## Virtual backgrounds and effects
+
+`createVideoEffectsPipeline(cameraTrack)` draws the camera through a canvas
+and hands back a processed track for `setVideoTrack`, so both transports
+publish it unchanged. Backgrounds: `blur`, animated procedural scenes
+(`aurora`, `cybergrid`, `bokeh`, `sunset`), or your own `image`. Overlays:
+`snow`, `confetti`, `vignette`. Person cut-out uses any `Segmenter` you plug
+in (for example a self-hosted MediaPipe selfie model); without one the
+pipeline degrades honestly — `blur` blurs the whole frame, scenes show the
+camera as a floating card.
+
+```ts
+const effects = createVideoEffectsPipeline(cameraTrack, { segmenter });
+effects.setBackground("aurora");
+effects.setOverlay("confetti");
+await media.setVideoTrack(effects.track);
+```
+
+## Virtual avatar
+
+`createAvatarPipeline(cameraTrack, { tracker })` analyzes the person on-device
+and publishes an animated avatar instead of the camera — blink, gaze, brows,
+jaw, mouth and head pose all mirror the real face. The output is an ordinary
+canvas track for `setVideoTrack`, so both transports carry it unchanged and
+the camera never leaves the machine.
+
+Face perception is pluggable: supply any `FaceTracker` (for example a
+self-hosted MediaPipe Face Landmarker emitting the 52 ARKit-style blendshape
+coefficients). Rendering is pluggable the same way — the built-in stylized
+renderer needs no assets, and a custom `AvatarRenderer` can draw a VRM
+character or anything else from the same `AvatarRig`.
+
+Two modes: `replace` (avatar on a backdrop) and `overlay` (avatar anchored to
+the tracked head over the live camera). Client environments differ, so the
+perception constants are an open `tuning` surface, adjustable live:
+
+```ts
+const avatar = createAvatarPipeline(cameraTrack, {
+  tracker,                 // your FaceTracker
+  mode: "replace",
+  tuning: { mouthGain: 1.7, eyeSync: 1, smoothing: 0.4 }
+});
+await media.setVideoTrack(avatar.track);
+avatar.setTuning({ mouthGain: 2 });   // cameras differ; correct per deployment
+avatar.setMode("overlay");
+```
+
+`AvatarTuning` covers `mouthGain`, `blinkGain`, `eyeSync` (evens the lids in
+profile, where per-eye data drifts), `smoothing`, `avatarScale`, `avatarLift`
+and `lostHoldMs` (losing the face holds the pose and eases the expression to
+rest instead of snapping to neutral).
+
+## Call recording
+
+`createCallRecorder(media.getNativeClient(), { credential })` composites every
+participant's video onto a canvas, mixes all audio, records with
+MediaRecorder, and on `stop()` uploads the file with the room signaling token.
+The platform stores it, bills the storage, lists it under
+`GET /v1/rtc/recordings` (Server API), and serves it from
+`GET /v1/rtc/recordings/{id}/download`. Recording happens on this client —
+what is recorded is what this participant saw and heard.
+
+```ts
+const recorder = createCallRecorder(media.getNativeClient(), { credential });
+await recorder.start();
+// ... later
+const { recordingId, durationSeconds } = await recorder.stop();
+```
+
 ## Active speaker
 
 The client samples WebRTC audio levels once a second on both transports and

package/dist/avatar-rig.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Face blendshapes → avatar rig.
+ *
+ * MediaPipe's Face Landmarker emits 52 ARKit-style blendshape coefficients
+ * and a facial transformation matrix. This module reduces them to the small
+ * rig a stylized avatar needs, as pure functions — the experimental Avatar
+ * Lab draws from this rig, and a future VRM renderer can consume the same
+ * values. Keeping it pure keeps it testable without a camera.
+ */
+export interface BlendshapeCategory {
+    categoryName: string;
+    score: number;
+}
+export interface AvatarRig {
+    /** 0 open .. 1 closed */
+    blinkLeft: number;
+    blinkRight: number;
+    /** 0 shut .. 1 wide open */
+    jawOpen: number;
+    /** -1 frown .. 1 smile */
+    smile: number;
+    /** 0 rest .. 1 raised */
+    browUp: number;
+    /** -1 left .. 1 right (viewer's perspective) */
+    pupilX: number;
+    /** -1 up .. 1 down */
+    pupilY: number;
+    /** radians */
+    headYaw: number;
+    headPitch: number;
+    headRoll: number;
+    /** whether a face was tracked at all */
+    tracked: boolean;
+}
+export declare function neutralRig(): AvatarRig;
+/**
+ * The 4x4 facial transformation matrix is column-major. Yaw/pitch/roll are
+ * extracted for a Y-up, viewer-facing convention; angles are what an avatar
+ * head joint wants directly.
+ */
+export declare function headPoseFromMatrix(matrix: ArrayLike<number> | null | undefined): {
+    yaw: number;
+    pitch: number;
+    roll: number;
+};
+export declare function rigFromBlendshapes(categories: BlendshapeCategory[] | null | undefined, matrix?: ArrayLike<number> | null, eyeSyncStrength?: number): AvatarRig;
+/**
+ * Exponential smoothing between detector ticks so the avatar renders
+ * smoothly even when detection runs slower than the paint loop.
+ */
+export declare function smoothRig(previous: AvatarRig, next: AvatarRig, alpha?: number): AvatarRig;
+/** Where to draw the head when compositing over a real frame. */
+export interface AvatarAnchor {
+    x: number;
+    y: number;
+    radius: number;
+}
+/**
+ * Draws the built-in stylized avatar. Pure canvas drawing, no assets. With an
+ * anchor the head is drawn over whatever is already on the canvas — the
+ * keep-real-background mode — otherwise it paints its own backdrop.
+ */
+export declare function drawAvatar(ctx: CanvasRenderingContext2D, rig: AvatarRig, width: number, height: number, anchor?: AvatarAnchor): void;

package/dist/avatar.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Virtual avatar pipeline.
+ *
+ * Same shape as the video-effects pipeline: the camera is analyzed on-device,
+ * an avatar canvas is captured as a MediaStreamTrack, and that track feeds the
+ * existing `setVideoTrack` — every transport (mesh or SFU) publishes the
+ * avatar without knowing it exists, and the network sees an ordinary video
+ * track.
+ *
+ * Face perception is an application asset, not an SDK constant: the pipeline
+ * takes any `FaceTracker` (the platform meeting room supplies a self-hosted
+ * MediaPipe Face Landmarker). Without a tracker the avatar idles untracked
+ * rather than pretending. Rendering is pluggable the same way: the built-in
+ * stylized renderer needs no assets, and a custom `AvatarRenderer` can draw
+ * anything driven by the same rig (the platform's VRM character renderer is
+ * exactly that).
+ *
+ * Client environments differ — cameras, lighting, faces — so every perception
+ * constant that needed hand-tuning during development is exposed in
+ * `AvatarTuning` and adjustable live via `setTuning`.
+ */
+import { type AvatarAnchor, type AvatarRig, type BlendshapeCategory } from "./avatar-rig";
+export interface FaceTrackerResult {
+    /** MediaPipe-style blendshape categories; null/empty means no face. */
+    categories: BlendshapeCategory[] | null;
+    /** Column-major 4x4 facial transformation matrix, when available. */
+    matrix?: ArrayLike<number> | null;
+    /** Normalized (0..1) face landmarks, used to anchor overlay mode. */
+    landmarks?: Array<{
+        x: number;
+        y: number;
+    }> | null;
+}
+export interface FaceTracker {
+    detect(frame: CanvasImageSource, timestampMs: number): FaceTrackerResult | null;
+    close?(): void;
+}
+export interface AvatarRenderer {
+    /** Draws the rig. `anchor` is set in overlay mode, null in replace mode. */
+    render(ctx: CanvasRenderingContext2D, rig: AvatarRig, width: number, height: number, anchor: AvatarAnchor | null): void;
+    dispose?(): void;
+}
+/** `replace`: avatar on a backdrop. `overlay`: avatar over the live camera, anchored to the tracked head. */
+export type AvatarMode = "replace" | "overlay";
+export interface AvatarTuning {
+    /** jawOpen multiplier — detectors top out well below 1 for a wide-open mouth. */
+    mouthGain: number;
+    /** Blink coefficient multiplier. */
+    blinkGain: number;
+    /** 0 off .. 1 full evening of the lids in profile (per-eye data drifts at high yaw). */
+    eyeSync: number;
+    /** Detector→rig easing per frame; lower is smoother but laggier. */
+    smoothing: number;
+    /** Avatar size relative to the tracked head (overlay mode). */
+    avatarScale: number;
+    /** Vertical offset in head radii, positive is down (overlay mode). */
+    avatarLift: number;
+    /** After this many ms without a face, the expression eases to rest (pose is held). */
+    lostHoldMs: number;
+}
+export declare const AVATAR_TUNING_DEFAULTS: AvatarTuning;
+export interface AvatarPipelineOptions {
+    width?: number;
+    height?: number;
+    frameRate?: number;
+    tracker?: FaceTracker | null;
+    renderer?: AvatarRenderer | null;
+    mode?: AvatarMode;
+    tuning?: Partial<AvatarTuning>;
+    /** Backdrop color for replace mode. */
+    backdrop?: string;
+    document?: Document;
+}
+export interface AvatarPipeline {
+    readonly track: MediaStreamTrack;
+    readonly stream: MediaStream;
+    /** The current smoothed rig — read-only, useful for meters and debugging. */
+    getRig(): AvatarRig;
+    setMode(mode: AvatarMode): void;
+    setTracker(tracker: FaceTracker | null): void;
+    setRenderer(renderer: AvatarRenderer | null): void;
+    setTuning(partial: Partial<AvatarTuning>): void;
+    getTuning(): AvatarTuning;
+    dispose(): void;
+}
+/** The built-in stylized renderer: pure canvas, no assets, no licensing. */
+export declare function stylizedAvatarRenderer(): AvatarRenderer;
+export declare function createAvatarPipeline(source: MediaStreamTrack | MediaStream, options?: AvatarPipelineOptions): AvatarPipeline;

package/dist/call-recorder.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Client-side call recording.
+ *
+ * The mesh has no server with access to the media, so the first recorder that
+ * can exist lives where the media already is: this client. It composites every
+ * video tile onto a canvas, mixes every audio track, records the result with
+ * MediaRecorder, and uploads the finished file with the room signaling token.
+ * The platform bills and purges the stored file through the same
+ * `rtc_recordings` rows a future server-side recorder will use.
+ */
+export interface RecordableRoomClient {
+    getRemoteStreams(): Array<{
+        participant: {
+            participantId: string;
+            displayName: string;
+        };
+        stream: MediaStream;
+    }>;
+    getLocalStream(): MediaStream | null;
+}
+export interface CallRecorderOptions {
+    credential: {
+        signalingToken: string;
+        signalingUrl: string;
+    };
+    /** Canvas output height; width follows 16:9. Default 720. */
+    height?: number;
+    mimeType?: string;
+    /** Test seams. */
+    document?: Document;
+    mediaRecorderFactory?: (stream: MediaStream, options: {
+        mimeType: string;
+    }) => MediaRecorder;
+    fetchImplementation?: typeof fetch;
+}
+export interface CallRecorderResult {
+    recordingId: string;
+    sizeBytes: number;
+    durationSeconds: number;
+}
+export interface CallRecorder {
+    readonly recording: boolean;
+    start(): Promise<void>;
+    /** Stops, uploads, and marks the recording ready. */
+    stop(): Promise<CallRecorderResult>;
+    dispose(): void;
+}
+export declare function createCallRecorder(client: RecordableRoomClient, options: CallRecorderOptions): CallRecorder;

package/dist/index.d.ts

@@ -2,6 +2,14 @@ export { createPictureInPictureController } from "./picture-in-picture";
 export type { PictureInPictureController, PictureInPictureOptions } from "./picture-in-picture";
 export { pickActiveSpeaker, SPEAKER_NOISE_FLOOR, SPEAKER_TAKEOVER_RATIO } from "./active-speaker";
 export type { SpeakerLevel } from "./active-speaker";
+export { createCallRecorder } from "./call-recorder";
+export type { CallRecorder, CallRecorderOptions, CallRecorderResult } from "./call-recorder";
+export { BACKGROUND_PRESETS, createVideoEffectsPipeline, OVERLAY_EFFECTS, scenePainters } from "./video-effects";
+export type { BackgroundPreset, OverlayEffect, Segmenter, VideoEffectsOptions, VideoEffectsPipeline } from "./video-effects";
+export { AVATAR_TUNING_DEFAULTS, createAvatarPipeline, stylizedAvatarRenderer } from "./avatar";
+export type { AvatarMode, AvatarPipeline, AvatarPipelineOptions, AvatarRenderer, AvatarTuning, FaceTracker, FaceTrackerResult } from "./avatar";
+export { drawAvatar, headPoseFromMatrix, neutralRig, rigFromBlendshapes, smoothRig } from "./avatar-rig";
+export type { AvatarAnchor, AvatarRig, BlendshapeCategory } from "./avatar-rig";
 export type RtcMode = "voice" | "video" | "meeting";
 /**
  * How media travels. `mesh` (the default) opens one connection per pair and
@@ -338,4 +346,4 @@ export declare class RtcIncomingClient extends EventTarget {
     private emit;
 }
 export declare function createRtcIncomingClient(options: RtcIncomingClientOptions): RtcIncomingClient;
-export declare const version = "1.3.14";
+export declare const version = "1.3.16";