@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.8

package/dist/components/LipEffect/settings.d.ts

@@ -0,0 +1,271 @@
+import { LipMaskMode } from './types';
+import { LipUvBounds } from './uvGuide';
+import * as THREE from "three";
+export declare const LIP_EFFECT_PRESET_KEYS: readonly ["lipstick", "gloss", "matte", "tint", "metallic", "gradient"];
+export type LipEffectPresetKey = (typeof LIP_EFFECT_PRESET_KEYS)[number];
+export type LipEffectPreset = LipEffectPresetKey | "custom";
+export declare const LIP_EFFECT_COLOR_MODES: readonly ["texture", "solid"];
+/** `texture` samples the design texture when one is given; `solid` ignores it. */
+export type LipEffectColorMode = (typeof LIP_EFFECT_COLOR_MODES)[number];
+export declare const LIP_EFFECT_GRADIENT_DIRECTIONS: readonly ["outward", "vertical", "horizontal"];
+/** `outward` runs from the mouth line to the outer edge; the others span the lip box. */
+export type LipEffectGradientDirection = (typeof LIP_EFFECT_GRADIENT_DIRECTIONS)[number];
+/**
+ * Serializable lip effect settings. Every field is a plain value so the set
+ * can live in a scene store and drive the `LipEffectSettingsPanel` directly.
+ *
+ * Texture transform fields act on the design texture around the lip region:
+ * scale is a multiplier, offset is a fraction of the lip region's size
+ * (`1` = one lip width / height), rotation is degrees counter-clockwise as
+ * seen on the face, and the flips mirror the design across the lip centre.
+ */
+export type LipEffectSettings = {
+    preset: LipEffectPreset;
+    maskMode: LipMaskMode;
+    /** `solid` paints procedurally (colour and gradient only), even if a design texture is set. */
+    colorMode: LipEffectColorMode;
+    /** Tint multiplied into the design (or the fill colour without a design texture). */
+    color: string;
+    opacity: number;
+    /** 0 shows the design as painted, 1 multiplies it fully by `color`. */
+    intensity: number;
+    /** How much the camera's own lip shading modulates the paint. 0 = flat. */
+    shading: number;
+    /** Specular highlight amount. */
+    gloss: number;
+    /** Highlight spread: 0 = tight mirror-like, 1 = broad and soft. */
+    roughness: number;
+    highlightColor: string;
+    /** Second colour blended in towards the outer lip edge by `gradientAmount`. */
+    gradientColor: string;
+    gradientAmount: number;
+    gradientDirection: LipEffectGradientDirection;
+    /** Feather width at the outer lip boundary as a fraction of the lip band. 0 = hard edge. */
+    edgeSoftness: number;
+    /** Separate tint for the inner lip (and the mouth interior in `outer-lips` mode). */
+    innerLipColor: string;
+    innerLipAmount: number;
+    /** How far the inner tint reaches out from the mouth line, as a fraction of the band. */
+    innerLipWidth: number;
+    /** Give the lower lip its own colour, opacity and (via `lowerDesignTexture`) texture. */
+    splitLips: boolean;
+    lowerLipColor: string;
+    lowerLipOpacity: number;
+    textureScaleX: number;
+    textureScaleY: number;
+    textureOffsetX: number;
+    textureOffsetY: number;
+    textureRotation: number;
+    textureFlipX: boolean;
+    textureFlipY: boolean;
+};
+export declare const DEFAULT_LIP_EFFECT_SETTINGS: LipEffectSettings;
+/**
+ * Built-in looks. A preset sets the material fields only; it never touches
+ * the mask mode or the texture transform, which belong to the design.
+ */
+export declare const LIP_EFFECT_PRESETS: Record<LipEffectPresetKey, Partial<LipEffectSettings>>;
+export declare const isLipEffectPresetKey: (value: unknown) => value is LipEffectPresetKey;
+/**
+ * Resolve partial settings to a complete, clamped set. Defaults come first,
+ * then the named preset (if any), then every explicitly provided value.
+ */
+export declare const resolveLipEffectSettings: (input?: Partial<LipEffectSettings>) => LipEffectSettings;
+/**
+ * Apply an edit. Choosing a different preset re-applies that preset's values,
+ * and only the fields the patch actually changes win over it. A panel sends
+ * the whole object, so unchanged fields there carry the previous look and
+ * must not override the preset; a sparse API patch keeps its explicit values.
+ */
+export declare const patchLipEffectSettings: (current: LipEffectSettings, patch: Partial<LipEffectSettings>) => LipEffectSettings;
+export declare const LIP_EFFECT_SETTINGS_FIELDS: [{
+    readonly key: "preset";
+    readonly label: "Preset";
+    readonly type: "select";
+    readonly options: [{
+        readonly label: "Custom";
+        readonly value: "custom";
+    }, ...{
+        label: string;
+        value: "gradient" | "tint" | "gloss" | "lipstick" | "matte" | "metallic";
+    }[]];
+}, {
+    readonly key: "maskMode";
+    readonly label: "Mask mode";
+    readonly type: "select";
+    readonly options: [{
+        readonly label: "Outer + inner lips (mouth cut out)";
+        readonly value: "outer-and-inner-lips";
+    }, {
+        readonly label: "Outer lips (mouth included)";
+        readonly value: "outer-lips";
+    }];
+}, {
+    readonly key: "colorMode";
+    readonly label: "Color mode";
+    readonly type: "select";
+    readonly options: [{
+        readonly label: "Design texture";
+        readonly value: "texture";
+    }, {
+        readonly label: "Procedural (solid / gradient)";
+        readonly value: "solid";
+    }];
+}, {
+    readonly key: "color";
+    readonly label: "Tint color";
+    readonly type: "color";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "opacity";
+    readonly label: "Opacity";
+    readonly type: "number";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "intensity";
+    readonly label: "Tint intensity";
+    readonly type: "number";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "shading";
+    readonly label: "Skin shading";
+    readonly type: "number";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "gloss";
+    readonly label: "Gloss";
+    readonly type: "number";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "roughness";
+    readonly label: "Roughness";
+    readonly type: "number";
+}, {
+    readonly key: "highlightColor";
+    readonly label: "Highlight color";
+    readonly type: "color";
+}, {
+    readonly key: "gradientColor";
+    readonly label: "Gradient color";
+    readonly type: "color";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "gradientAmount";
+    readonly label: "Gradient amount";
+    readonly type: "number";
+}, {
+    readonly key: "gradientDirection";
+    readonly label: "Gradient direction";
+    readonly type: "select";
+    readonly options: [{
+        readonly label: "Mouth line → outer edge";
+        readonly value: "outward";
+    }, {
+        readonly label: "Bottom → top";
+        readonly value: "vertical";
+    }, {
+        readonly label: "Left → right";
+        readonly value: "horizontal";
+    }];
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "edgeSoftness";
+    readonly label: "Edge softness";
+    readonly type: "number";
+}, {
+    readonly key: "innerLipColor";
+    readonly label: "Inner lip color";
+    readonly type: "color";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "innerLipAmount";
+    readonly label: "Inner lip amount";
+    readonly type: "number";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "innerLipWidth";
+    readonly label: "Inner lip width";
+    readonly type: "number";
+}, {
+    readonly key: "splitLips";
+    readonly label: "Separate lower lip";
+    readonly type: "boolean";
+}, {
+    readonly key: "lowerLipColor";
+    readonly label: "Lower lip color";
+    readonly type: "color";
+}, {
+    readonly min: 0;
+    readonly max: 1;
+    readonly step: 0.01;
+    readonly key: "lowerLipOpacity";
+    readonly label: "Lower lip opacity";
+    readonly type: "number";
+}, {
+    readonly key: "textureScaleX";
+    readonly label: "Texture scale X";
+    readonly type: "number";
+    readonly min: 0.1;
+    readonly max: 10;
+    readonly step: 0.05;
+}, {
+    readonly key: "textureScaleY";
+    readonly label: "Texture scale Y";
+    readonly type: "number";
+    readonly min: 0.1;
+    readonly max: 10;
+    readonly step: 0.05;
+}, {
+    readonly key: "textureOffsetX";
+    readonly label: "Texture offset X";
+    readonly type: "number";
+    readonly min: -1;
+    readonly max: 1;
+    readonly step: 0.01;
+}, {
+    readonly key: "textureOffsetY";
+    readonly label: "Texture offset Y";
+    readonly type: "number";
+    readonly min: -1;
+    readonly max: 1;
+    readonly step: 0.01;
+}, {
+    readonly key: "textureRotation";
+    readonly label: "Texture rotation°";
+    readonly type: "number";
+    readonly min: -180;
+    readonly max: 180;
+    readonly step: 1;
+}, {
+    readonly key: "textureFlipX";
+    readonly label: "Flip horizontal";
+    readonly type: "boolean";
+}, {
+    readonly key: "textureFlipY";
+    readonly label: "Flip vertical";
+    readonly type: "boolean";
+}];
+/**
+ * The UV-space sampling matrix for the design texture: `designUv = M * [u, v, 1]`.
+ * It is the inverse of the visual transform (scale, rotate, offset, flip
+ * about the lip region centre), so the design moves the way the settings read.
+ */
+export declare const createLipTextureTransform: (settings: Pick<LipEffectSettings, "textureScaleX" | "textureScaleY" | "textureOffsetX" | "textureOffsetY" | "textureRotation" | "textureFlipX" | "textureFlipY">, bounds: LipUvBounds, target?: THREE.Matrix3) => THREE.Matrix3;

package/dist/components/LipEffect/types.d.ts

@@ -1,22 +1,69 @@
 import { ReactNode } from 'react';
+import { LipEffectSettings } from './settings';
+import { LipUvBounds } from './uvGuide';
 import type * as THREE from "three";
 export type LipMaskMode = "outer-lips" | "outer-and-inner-lips";
+/**
+ * Stable inputs handed to a custom lip material factory. The default
+ * `LipEffectMaterial` is built from exactly these.
+ */
 export type LipEffectMaterialContext = {
+    /** Live camera frame; sample it with `faceUv` to read the skin under the lips. */
     cameraTexture: THREE.Texture | null;
+    /** The creator's design texture in canonical face UV space, or `null` when none was given. */
     designTexture: THREE.Texture | null;
-    lipMask: THREE.Texture | null;
-    faceUv: THREE.BufferAttribute | null;
+    /**
+     * Live per-vertex camera-texture UVs. The same attribute is bound to the lip
+     * geometry as `attribute vec2 faceUv`, so a custom vertex shader can read it
+     * directly; the object is exposed for uniforms, debugging, or readback.
+     */
+    faceUv: THREE.BufferAttribute;
+    /** Static per-vertex lip band position, bound as `attribute float lipEdge` (0 outer ring, 1 mouth line). */
+    lipEdge: THREE.BufferAttribute;
+    /** Design texture for the lower lip when `splitLips` is on, or `null`. */
+    lowerDesignTexture: THREE.Texture | null;
     faceIndex: number;
+    /** The active mask mode, so a custom material can decide whether to honour the mouth stencil. */
+    maskMode: LipMaskMode;
+    /** Lip region of the canonical UV atlas, for texture transforms and gradients. */
+    lipUvBounds: LipUvBounds;
+    /** The resolved settings driving the built-in material. */
+    settings: LipEffectSettings;
 };
-export type LipEffectMaterialSpec = ReactNode | ((context: LipEffectMaterialContext) => ReactNode);
-export type LipEffectProps = {
+/** A material node, or a three.js material instance (which can be shared across meshes and faces). */
+export type LipEffectMaterialResult = ReactNode | THREE.Material;
+export type LipEffectMaterialSpec = LipEffectMaterialResult | ((context: LipEffectMaterialContext) => LipEffectMaterialResult);
+export type LipEffectProps = Partial<LipEffectSettings> & {
     faceIndex?: number;
+    /** Design texture in canonical face UV space. Without one the lips are filled with `color`. */
     designTexture?: THREE.Texture | string;
-    color?: string;
-    opacity?: number;
-    intensity?: number;
-    maskMode?: LipMaskMode;
-    shading?: boolean;
+    /** Lower-lip design texture, used when `splitLips` is on. Falls back to `designTexture`. */
+    lowerDesignTexture?: THREE.Texture | string;
     material?: LipEffectMaterialSpec;
+    /**
+     * Use this material instead of the built-in one (and instead of `material`),
+     * for example one from `useLipEffectMaterial` shared by several faces.
+     */
+    sharedMaterial?: THREE.Material;
+    /** Cap how often the lip mesh follows the landmarks; omit to update every frame. */
+    maxUpdateFps?: number;
+    /** Keep the lips on the last landmarks for this long (ms) across tracker dropouts. Default 250. */
+    holdMs?: number;
+    /** While held, extrapolate the landmarks along their last velocity for up to this long (ms). Default 0. */
+    predictMs?: number;
+    renderOrder?: number;
+};
+/** Per-face configuration for `LipEffects`: everything but the face index. */
+export type LipEffectFaceConfig = Omit<LipEffectProps, "faceIndex">;
+export type LipEffectsProps = {
+    /** How many face slots to render. Match the tracker's `numFaces`. Default 2. */
+    maxFaces?: number;
+    /** One config for all faces, or a picker per face (`null` skips a face). */
+    faces: LipEffectFaceConfig | ((faceIndex: number) => LipEffectFaceConfig | null);
+    /** Share one built-in material across faces. Default: on for a single config, off for a picker. */
+    shareMaterial?: boolean;
+    maxUpdateFps?: number;
+    holdMs?: number;
+    predictMs?: number;
     renderOrder?: number;
 };

package/dist/components/LipEffect/useLipDesignTextures.d.ts

@@ -0,0 +1,11 @@
+import * as THREE from "three";
+export type LipDesignTextureSource = THREE.Texture | string | undefined;
+/**
+ * Resolve the design texture props (URL or texture object) to textures.
+ * URLs load through R3F's loader cache, so faces sharing a URL share one
+ * texture; a constant hook count keeps the pair reloadable.
+ */
+export declare const useLipDesignTextures: (designTexture: LipDesignTextureSource, lowerDesignTexture: LipDesignTextureSource) => {
+    designTexture: THREE.Texture | null;
+    lowerDesignTexture: THREE.Texture | null;
+};

package/dist/components/LipEffect/uvGuide.d.ts

@@ -0,0 +1,59 @@
+import { ParsedCanonicalMesh } from '../FaceMesh/canonical/parseCanonicalFaceModel';
+/** Lip extent in canonical UV space (`v` grows upward, as in the OBJ). */
+export type LipUvBounds = {
+    minU: number;
+    maxU: number;
+    minV: number;
+    maxV: number;
+    /** `v` of the mouth line (mean of the inner ring); above it is the upper lip. */
+    mouthV: number;
+};
+/**
+ * Lip region of the canonical mesh that ships with the SDK, in UV space.
+ * Used until the mesh has loaded; `getLipUvBounds(mesh)` is authoritative.
+ */
+export declare const CANONICAL_LIP_UV_BOUNDS: LipUvBounds;
+export type LipUvGuideOptions = {
+    /** Pixel size of the square texture canvas the guide represents. Default 1024. */
+    size?: number;
+    /** Frame the whole canvas (the design contract) or zoom onto the lip region. Default "canvas". */
+    region?: "canvas" | "lips";
+    /** Draw the lip triangle wireframe. Default true. */
+    wireframe?: boolean;
+    /** Draw orientation and region labels. Default true. */
+    labels?: boolean;
+    /** Fill the canvas with a colour. Omit for a transparent background. */
+    background?: string;
+};
+export declare const LIP_UV_GUIDE_COLORS: {
+    outerRing: string;
+    innerRing: string;
+    wireframe: string;
+    text: string;
+    frame: string;
+};
+/**
+ * Convert a canonical UV to texture pixel space. `TextureLoader` textures are
+ * `flipY`, so `v = 1` is the top row of the image.
+ */
+export declare const lipUvToPixel: (u: number, v: number, size: number) => {
+    x: number;
+    y: number;
+};
+/** Bounding box of the outer lip ring in canonical UV space, plus the mouth line. */
+export declare const getLipUvBounds: (mesh: ParsedCanonicalMesh) => LipUvBounds;
+/**
+ * Render a creator-facing lip UV template as an SVG string, derived from the
+ * mesh's own `uv` attribute. It shows the texture canvas, the outer-lip and
+ * inner-mouth rings, the lip triangles that are actually sampled, and which
+ * side of the image lands on which side of the subject's mouth.
+ */
+export declare const createLipUvGuideSvg: (mesh: ParsedCanonicalMesh, options?: LipUvGuideOptions) => string;
+/** The guide as an `image/svg+xml` data URL, loadable with `THREE.TextureLoader` or an `<img>`. */
+export declare const createLipUvGuideDataUrl: (mesh: ParsedCanonicalMesh, options?: LipUvGuideOptions) => string;
+/**
+ * Rasterize the guide to a PNG `Blob` in the browser (an `<img>` and a 2D
+ * canvas), for download or for use as a texture. Transparent unless
+ * `options.background` is set.
+ */
+export declare const createLipUvGuidePngBlob: (mesh: ParsedCanonicalMesh, options?: LipUvGuideOptions) => Promise<Blob>;

package/dist/components/PersonCutout/index.d.ts

@@ -0,0 +1,50 @@
+import { ScreenImageProps } from '../ScreenImage/types';
+import { PersonCutoutFallback, PersonCutoutStatus } from './personCutout';
+import type * as THREE from "three";
+export type { PersonCutoutFallback, PersonCutoutStatus } from './personCutout';
+export { FULL_SCREEN_ANCHORS, resolvePersonCutoutStatus, shouldDrawFallback } from './personCutout';
+export type PersonMaskTextureResult = {
+    /**
+     * The portrait segmentation mask in feed space. The signal is in RGB:
+     * white (255) on the person, black (0) on the background, alpha 255
+     * everywhere (measured; three's `alphaMap` reads green, which is how
+     * `VideoBackground` consumes it).
+     */
+    texture: THREE.Texture | null;
+    status: PersonCutoutStatus;
+    error: Error | null;
+};
+/**
+ * The portrait segmentation mask as a texture, with the model's lifecycle.
+ * Acquires the segmentation model itself. The mask lives in feed space, like
+ * the camera texture; read its green channel: 255 on the person, 0 elsewhere.
+ */
+export declare const usePersonMaskTexture: (active?: boolean) => PersonMaskTextureResult;
+export type PersonCutoutProps = Omit<ScreenImageProps, "source" | "src" | "maskTexture" | "maskChannel" | "maskInvert"> & {
+    /** Edge softening as a 3×3 blur radius in mask texels. Default 4; 0 = hard edge. */
+    maskFeather?: number;
+    /**
+     * Shrink the kept silhouette into the feathered edge (0..1 of the feather
+     * width) so background fringe from the segmenter is trimmed. Default 0.7.
+     */
+    maskErode?: number;
+    /** Show the background only (cut the person out) instead of the person only. */
+    invert?: boolean;
+    /**
+     * What to draw while the segmentation model is loading or has failed:
+     * the unmasked camera (default, the experience degrades to a plain feed)
+     * or nothing.
+     */
+    fallback?: PersonCutoutFallback;
+    /** Keep the segmentation model running. Default true. */
+    active?: boolean;
+    onStatusChange?: (status: PersonCutoutStatus, error: Error | null) => void;
+};
+/**
+ * The live camera, masked to the person, as a full-screen `ScreenImage`.
+ * Place any `ScreenImage` (image, video, GIF) or shader surface behind it
+ * with a lower `renderOrder` for background replacement. The camera texture is
+ * already cover-cropped and webcam-mirrored by the session, and the mask is in
+ * that same space, so no coordinate work is needed.
+ */
+export declare const PersonCutout: ({ invert, fallback, active, onStatusChange, fit, anchors, renderOrder, visible, maskFeather, maskErode, ...screenImageProps }: PersonCutoutProps) => import("react/jsx-runtime").JSX.Element | null;

package/dist/components/PersonCutout/personCutout.d.ts

@@ -0,0 +1,16 @@
+import { ScreenAnchorEdges } from '../ScreenSpaceUI';
+/** Anchors that stretch a screen element over the whole viewport. */
+export declare const FULL_SCREEN_ANCHORS: ScreenAnchorEdges;
+export type PersonCutoutStatus = "loading" | "ready" | "error";
+/**
+ * Status of a person cut-out from the segmentation model's lifecycle and
+ * whether its mask texture exists yet.
+ */
+export declare const resolvePersonCutoutStatus: ({ ready, error, hasMask, }: {
+    ready: boolean;
+    error: Error | null;
+    hasMask: boolean;
+}) => PersonCutoutStatus;
+export type PersonCutoutFallback = "camera" | "hidden";
+/** Whether the camera layer should draw while the mask is not available. */
+export declare const shouldDrawFallback: (status: PersonCutoutStatus, fallback: PersonCutoutFallback) => boolean;

package/dist/components/ScreenImage/components/ScreenImageBody.d.ts

@@ -5,5 +5,5 @@ type ScreenImageBodyProps = Omit<ScreenImageProps, "anchors" | "coordinateSpace"
         height: number;
     };
 };
-export declare function ScreenImageBody({ name, source, src, fit, stretchMode, materialType, blendMode, backdropTexture: providedBackdropTexture, rotation, opacity, color, flipX, flipY, renderOrder, transparent, alphaTest, overlay, visible, layoutSize, }: ScreenImageBodyProps): import("react/jsx-runtime").JSX.Element;
+export declare function ScreenImageBody({ name, source, src, fit, stretchMode, materialType, blendMode, backdropTexture: providedBackdropTexture, rotation, opacity, color, maskTexture, maskChannel, maskInvert, maskFeather, maskErode, flipX, flipY, renderOrder, transparent, alphaTest, overlay, visible, layoutSize, }: ScreenImageBodyProps): import("react/jsx-runtime").JSX.Element;
 export {};

package/dist/components/ScreenImage/index.d.ts

@@ -1,8 +1,10 @@
 export { ScreenImage, defaultScreenImageUrl } from './ScreenImage';
 export { screenImageBlendModeOptions } from './utils/ScreenImageConstants';
 export { resolveBlendProps } from './materials/ScreenImageBlendMaterial';
+export { SCREEN_IMAGE_MASK_CHANNELS, resolveMaskChannelIndex, } from './materials/ScreenImageMaskMaterial';
+export type { ScreenImageMaskChannel } from './materials/ScreenImageMaskMaterial';
 export type { ScreenImageBlendMode, ScreenImageMaterialType, ScreenImageProps } from './types';
 export type { ScreenImageSource } from './source';
-export { getScreenImageTextureAspect, getScreenImageTextureSize, resolveScreenImageSource, } from './source';
+export { getScreenImageTextureAspect, getScreenImageTextureSize, isDynamicScreenImageTexture, resolveScreenImageSource, } from './source';
 export { computeScreenImageFitResolution, computeScreenImageFittedRect, resolveScreenImageStretchMode, } from './fit';
 export type { ScreenImageFit, ScreenImageFittedRect, ScreenImageFittedRectInput, ScreenImageStretchMode, } from './fit';

package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts

@@ -6,4 +6,4 @@ export declare const advancedBlendModeIndices: Record<AdvancedBlendMode, number>
 export type BlendMaterialProps = Partial<Pick<THREE.Material, "blending" | "blendSrc" | "blendDst" | "blendEquation">>;
 export declare const resolveBlendProps: (mode: ScreenImageBlendMode) => BlendMaterialProps;
 export declare const advancedBlendVertexShader = "\nvarying vec2 vUv;\nvoid main(){vUv=uv;gl_Position=projectionMatrix*modelViewMatrix*vec4(position,1.0);}";
-export declare const advancedBlendFragmentShader = "\nuniform sampler2D imageMap;uniform sampler2D backdropMap;uniform vec2 resolution;\nuniform vec3 tint;uniform float opacity;uniform float alphaTest;uniform int blendMode;varying vec2 vUv;\nfloat overlayChannel(float b,float s){return b<.5?2.*b*s:1.-2.*(1.-b)*(1.-s);}\nfloat hardLightChannel(float b,float s){return s<.5?2.*b*s:1.-2.*(1.-b)*(1.-s);}\nfloat softLightChannel(float b,float s){return s<.5?b-(1.-2.*s)*b*(1.-b):b+(2.*s-1.)*(sqrt(max(b,0.))-b);}\nvec3 applyAdvancedBlend(vec3 b,vec3 s){\nif(blendMode==0)return vec3(overlayChannel(b.r,s.r),overlayChannel(b.g,s.g),overlayChannel(b.b,s.b));\nif(blendMode==1)return vec3(softLightChannel(b.r,s.r),softLightChannel(b.g,s.g),softLightChannel(b.b,s.b));\nif(blendMode==2)return vec3(hardLightChannel(b.r,s.r),hardLightChannel(b.g,s.g),hardLightChannel(b.b,s.b));\nif(blendMode==3)return abs(b-s);return b+s-2.*b*s;}\nvoid main(){vec4 image=texture2D(imageMap,vUv)*vec4(tint,opacity);if(image.a<=alphaTest)discard;\nvec2 uv=gl_FragCoord.xy/max(resolution,vec2(1.));vec3 base=texture2D(backdropMap,uv).rgb;\nvec3 mixed=applyAdvancedBlend(base,image.rgb);gl_FragColor=vec4(mix(base,mixed,image.a),1.);}";
+export declare const advancedBlendFragmentShader = "\nuniform sampler2D imageMap;uniform sampler2D backdropMap;uniform vec2 resolution;\nuniform vec3 tint;uniform float opacity;uniform float alphaTest;uniform int blendMode;varying vec2 vUv;\nfloat overlayChannel(float b,float s){return b<.5?2.*b*s:1.-2.*(1.-b)*(1.-s);}\nfloat hardLightChannel(float b,float s){return s<.5?2.*b*s:1.-2.*(1.-b)*(1.-s);}\nfloat softLightChannel(float b,float s){return s<.5?b-(1.-2.*s)*b*(1.-b):b+(2.*s-1.)*(sqrt(max(b,0.))-b);}\nvec3 applyAdvancedBlend(vec3 b,vec3 s){\nif(blendMode==0)return vec3(overlayChannel(b.r,s.r),overlayChannel(b.g,s.g),overlayChannel(b.b,s.b));\nif(blendMode==1)return vec3(softLightChannel(b.r,s.r),softLightChannel(b.g,s.g),softLightChannel(b.b,s.b));\nif(blendMode==2)return vec3(hardLightChannel(b.r,s.r),hardLightChannel(b.g,s.g),hardLightChannel(b.b,s.b));\nif(blendMode==3)return abs(b-s);return b+s-2.*b*s;}\nvoid main(){vec4 image=texture2D(imageMap,vUv)*vec4(tint,opacity);if(image.a<=alphaTest)discard;\nvec2 uv=gl_FragCoord.xy/max(resolution,vec2(1.));vec3 base=texture2D(backdropMap,uv).rgb;\nvec3 mixed=applyAdvancedBlend(base,image.rgb);gl_FragColor=vec4(mix(base,mixed,image.a),1.);\n#include <colorspace_fragment>\n}";

package/dist/components/ScreenImage/materials/ScreenImageMaskMaterial.d.ts

@@ -0,0 +1,11 @@
+/** Which channel of a `maskTexture` carries coverage. */
+export declare const SCREEN_IMAGE_MASK_CHANNELS: readonly ["alpha", "red", "green", "luminance"];
+export type ScreenImageMaskChannel = (typeof SCREEN_IMAGE_MASK_CHANNELS)[number];
+export declare const resolveMaskChannelIndex: (channel: ScreenImageMaskChannel | undefined) => number;
+export declare const maskedVertexShader = "\nvarying vec2 vUv;\nvoid main(){vUv=uv;gl_Position=projectionMatrix*modelViewMatrix*vec4(position,1.0);}";
+/**
+ * Image × mask. Both are sampled through the image's fit transform (repeat /
+ * offset / flips), so a mask authored in the image's own space — a camera
+ * frame and its segmentation mask — stays aligned under cover-cropping.
+ */
+export declare const maskedFragmentShader = "\nuniform sampler2D imageMap;uniform sampler2D maskMap;uniform mat3 mapTransform;\nuniform vec3 tint;uniform float opacity;uniform float alphaTest;uniform int maskChannel;uniform float maskInvert;\nuniform float maskFeather;uniform float maskErode;uniform vec2 maskTexel;\nvarying vec2 vUv;\nfloat coverageAt(vec2 uv){\nvec4 m=texture2D(maskMap,uv);\nfloat c=maskChannel==0?m.a:maskChannel==1?m.r:maskChannel==2?m.g:dot(m.rgb,vec3(0.299,0.587,0.114));\nreturn mix(c,1.0-c,maskInvert);}\nvoid main(){\nvec2 uv=(mapTransform*vec3(vUv,1.0)).xy;\nvec4 image=texture2D(imageMap,uv);\nfloat coverage;\nif(maskFeather>0.0){\nvec2 step=maskTexel*maskFeather;\ncoverage=0.0;\nfor(int y=-1;y<=1;y++){for(int x=-1;x<=1;x++){coverage+=coverageAt(uv+vec2(float(x),float(y))*step);}}\ncoverage/=9.0;\n// Erode: raise the threshold on the blurred edge so the silhouette shrinks\n// by up to the feather radius (maskErode is a fraction of that width).\ncoverage=smoothstep(clamp(maskErode,0.0,0.95),1.0,coverage);\n}else{coverage=coverageAt(uv);}\nfloat alpha=image.a*opacity*coverage;\nif(alpha<=alphaTest)discard;\ngl_FragColor=vec4(image.rgb*tint,alpha);\n#include <colorspace_fragment>\n}";

package/dist/components/ScreenImage/source.d.ts

@@ -16,4 +16,10 @@ export declare const getScreenImageTextureSize: (texture: THREE.Texture) => {
     width: number;
     height: number;
 };
+/**
+ * Textures whose pixels change after upload: videos and any canvas-backed
+ * texture, including `OffscreenCanvas` sources such as the session camera
+ * feed. These need `needsUpdate` every frame to stay live.
+ */
+export declare const isDynamicScreenImageTexture: (texture: THREE.Texture) => boolean;
 export declare const getScreenImageTextureAspect: (texture: THREE.Texture) => number;

package/dist/components/ScreenImage/types.d.ts

@@ -1,6 +1,7 @@
 import { ScreenAnchorEdges, ScreenOffsetEdges, ScreenTransformGuideLayer } from '../ScreenSpaceUI';
 import { ScreenCanvasSize, ScreenCoordinateSpace, ScreenTransform2DSettings } from '../../screen2d';
 import { ScreenImageFit, ScreenImageStretchMode } from './fit';
+import { ScreenImageMaskChannel } from './materials/ScreenImageMaskMaterial';
 import { ScreenImageSource } from './source';
 import type * as THREE from "three";
 export type ScreenImageMaterialType = "unlit" | "standardPBR" | "matcap";
@@ -27,6 +28,23 @@ export type ScreenImageProps = {
     backdropTexture?: THREE.Texture | null;
     opacity?: number;
     color?: THREE.ColorRepresentation;
+    /**
+     * Alpha mask in the image's own texture space: it is sampled through the
+     * same fit / crop / flip transform as the image, so a camera frame and its
+     * segmentation mask stay aligned. Takes precedence over advanced blend modes.
+     */
+    maskTexture?: THREE.Texture | null;
+    /** Channel of `maskTexture` that carries coverage. Default `"alpha"`. */
+    maskChannel?: ScreenImageMaskChannel;
+    /** Keep where the mask is 0 instead of 1. */
+    maskInvert?: boolean;
+    /** Soften mask edges: 3×3 blur radius in mask texels. 0 = hard. */
+    maskFeather?: number;
+    /**
+     * Shrink the kept region into the feathered edge, 0..1 of the feather
+     * width; removes fringe from the source. Needs `maskFeather` > 0.
+     */
+    maskErode?: number;
     flipX?: boolean;
     flipY?: boolean;
     renderOrder?: number;

package/dist/components/SegmentationOverlay/SegmentationOverlay.d.ts

@@ -0,0 +1,37 @@
+import { ReactNode } from 'react';
+import { SemanticMask } from '../../segmentation2d/types';
+import { SegmentationOverlaySettings } from './settings';
+import * as THREE from "three";
+export type SegmentationOverlayMaterialContext = {
+    cameraTexture: THREE.Texture | null;
+    /**
+     * Class-index texture (red channel, 0..255 → index); nearest-filtered.
+     * Re-allocated when the mask changes size; the factory is re-run with the
+     * new texture, so read it from the context rather than caching it.
+     */
+    maskTexture: THREE.Texture;
+    /** Feed UV → mask UV, including the row flip, mirror and source cover-fit. */
+    maskTransform: THREE.Matrix3;
+    /** Class indices the overlay selects, resolved from `label` against the mask; empty until the mask has data. */
+    labelIndices: number[];
+    mask: SemanticMask | null;
+    settings: SegmentationOverlaySettings;
+};
+export type SegmentationOverlayMaterialSpec = ReactNode | THREE.Material | ((context: SegmentationOverlayMaterialContext) => ReactNode | THREE.Material);
+export type SegmentationOverlayProps = Partial<SegmentationOverlaySettings> & {
+    /** The mask to draw from: a provider hook's result or your own `SemanticMask`. */
+    mask: SemanticMask | null;
+    /** Optional image multiplied into the paint, sampled in screen UV (cover of the viewport). */
+    texture?: THREE.Texture | string;
+    /** Custom material (node, instance, or factory). Replaces the built-in look. */
+    material?: SegmentationOverlayMaterialSpec;
+    /** Default: just above the video background. */
+    renderOrder?: number;
+    visible?: boolean;
+};
+/**
+ * Draw one or more semantic regions of a mask as a screen-aligned 2D layer,
+ * tinted and composited over the camera feed. Covers the viewport exactly
+ * like `VideoBackground`, so feed-space masks need no coordinate work.
+ */
+export declare const SegmentationOverlay: ({ mask, texture, material, renderOrder, visible, ...settingsInput }: SegmentationOverlayProps) => import("react/jsx-runtime").JSX.Element | null;

package/dist/components/SegmentationOverlay/SegmentationOverlayMaterial.d.ts

@@ -0,0 +1,31 @@
+import { SemanticMask } from '../../segmentation2d/types';
+import { SegmentationOverlaySettings } from './settings';
+import * as THREE from "three";
+/** Up to this many labels can be selected by one overlay. */
+export declare const MAX_OVERLAY_LABELS = 4;
+export type SegmentationOverlayMaterialInputs = {
+    cameraTexture: THREE.Texture | null;
+    maskTexture: THREE.Texture;
+    mask: SemanticMask | null;
+    settings: SegmentationOverlaySettings;
+    /** Optional image sampled in screen UV and multiplied into the paint. */
+    designTexture?: THREE.Texture | null;
+    /** Feed (camera texture) size, for cover-fitting source-space masks. */
+    feedWidth?: number;
+    feedHeight?: number;
+};
+export declare const SEGMENTATION_OVERLAY_VERTEX_SHADER = "\nvarying vec2 vUv;\nvoid main() {\n  vUv = uv;\n  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);\n}";
+export declare const SEGMENTATION_OVERLAY_FRAGMENT_SHADER = "\nuniform sampler2D cameraTexture;\nuniform sampler2D maskTexture;\nuniform sampler2D designTexture;\nuniform float useDesign;\nuniform vec4 labels;\nuniform float labelCount;\nuniform float invert;\nuniform vec3 tint;\nuniform float opacity;\nuniform float blendMode;\nuniform float feather;\nuniform vec2 maskTexel;\nuniform mat3 maskTransform;\nvarying vec2 vUv;\n\nfloat member(vec2 uv) {\n  if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) return 0.0;\n  float index = floor(texture2D(maskTexture, uv).r * 255.0 + 0.5);\n  float m = 0.0;\n  if (labelCount > 0.5 && abs(index - labels.x) < 0.5) m = 1.0;\n  if (labelCount > 1.5 && abs(index - labels.y) < 0.5) m = 1.0;\n  if (labelCount > 2.5 && abs(index - labels.z) < 0.5) m = 1.0;\n  if (labelCount > 3.5 && abs(index - labels.w) < 0.5) m = 1.0;\n  return invert > 0.5 ? 1.0 - m : m;\n}\n\nvoid main() {\n  vec2 maskUv = (maskTransform * vec3(vUv, 1.0)).xy;\n  float coverage;\n  if (feather > 0.0) {\n    vec2 step = maskTexel * feather;\n    coverage = 0.0;\n    for (int y = -1; y <= 1; y++) {\n      for (int x = -1; x <= 1; x++) {\n        coverage += member(maskUv + vec2(float(x), float(y)) * step);\n      }\n    }\n    coverage /= 9.0;\n  } else {\n    coverage = member(maskUv);\n  }\n\n  vec3 base = texture2D(cameraTexture, vUv).rgb;\n  vec4 design = useDesign > 0.5 ? texture2D(designTexture, vUv) : vec4(1.0);\n  vec3 paint = tint * design.rgb;\n  vec3 blended = blendMode < 0.5\n    ? paint\n    : blendMode < 1.5\n      ? base * paint\n      : blendMode < 2.5\n        ? 1.0 - (1.0 - base) * (1.0 - paint)\n        : mix(2.0 * base * paint, 1.0 - 2.0 * (1.0 - base) * (1.0 - paint), step(0.5, base));\n\n  gl_FragColor = vec4(blended, coverage * opacity * design.a);\n  #include <colorspace_fragment>\n}";
+/** The built-in overlay material as a plain three.js object (shareable across meshes). */
+export declare const createSegmentationOverlayMaterial: () => THREE.ShaderMaterial;
+/** Resolve the selected labels to class indices for a mask (unknown labels are skipped). */
+export declare const resolveOverlayLabelIndices: (mask: Pick<SemanticMask, "labels"> | null, label: string | readonly string[]) => number[];
+/**
+ * The class indices an overlay should draw right now: the resolved labels,
+ * or none while the mask is still at `version` 0 (no data yet). Shared by
+ * the built-in material and the custom-material context so both agree.
+ */
+export declare const resolveActiveOverlayLabelIndices: (mask: SemanticMask | null, label: string | readonly string[]) => number[];
+export declare const updateSegmentationOverlayMaterial: (material: THREE.ShaderMaterial, { cameraTexture, maskTexture, mask, settings, designTexture, feedWidth, feedHeight, }: SegmentationOverlayMaterialInputs) => THREE.ShaderMaterial;
+/** Own one overlay material and keep it in sync with the inputs on every render. */
+export declare const useSegmentationOverlayMaterial: (inputs: SegmentationOverlayMaterialInputs) => THREE.ShaderMaterial;