@vincentt-xr/sdk 2.0.0-alpha.0 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GROUNDING.md +1813 -27
- package/dist/{DebuggerUI-CEfdddkZ.cjs → DebuggerUI-B45_Gb9d.cjs} +1 -1
- package/dist/{DebuggerUI-Dgop6Q6o.js → DebuggerUI-Cq28c18s.js} +1 -1
- package/dist/FaceMeshRenderer-6t8OsQhK.js +1195 -0
- package/dist/FaceMeshRenderer-lQLhYIW-.cjs +155 -0
- package/dist/FaceRetouch-CDrL0pXg.js +551 -0
- package/dist/FaceRetouch-DtNl8-4e.cjs +134 -0
- package/dist/audio/AudioEngine.d.ts +3 -1
- package/dist/audio/AudioPlayer.d.ts +29 -0
- package/dist/audio/AudioPlayerLogic.d.ts +4 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/components/Axes/index.d.ts +6 -0
- package/dist/components/FaceMesh/FaceMeshRenderer.d.ts +17 -0
- package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +22 -1
- package/dist/components/FaceMesh/canonical/FaceMeshMaterial.d.ts +25 -0
- package/dist/components/FaceMesh/canonical/assets.d.ts +3 -0
- package/dist/components/FaceMesh/canonical/geometryGroups.d.ts +9 -0
- package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +2 -0
- package/dist/components/FaceMesh/canonical/shaders.d.ts +2 -0
- package/dist/components/FaceMesh/deform.d.ts +3 -0
- package/dist/components/FaceMesh/index.d.ts +5 -0
- package/dist/components/FaceMesh/material/defaults.d.ts +74 -0
- package/dist/components/FaceMesh/material/index.d.ts +1 -0
- package/dist/components/FaceMesh/material/options.d.ts +5 -1
- package/dist/components/FaceMesh/material/support.d.ts +7 -2
- package/dist/components/FaceMesh/material/textures.d.ts +4 -0
- package/dist/components/FaceMesh/material/types.d.ts +4 -2
- package/dist/components/FaceRetouch/Background.d.ts +9 -5
- package/dist/components/FaceRetouch/FaceRetouch.d.ts +47 -0
- package/dist/components/FaceRetouch/index.d.ts +2 -6
- package/dist/components/FaceRetouch/landmarkMask.d.ts +4 -6
- package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +4 -7
- package/dist/components/FaceTracker/index.d.ts +5 -3
- package/dist/components/FaceWarp3D/FaceWarp3DHead.d.ts +15 -0
- package/dist/components/FaceWarp3D/defaults.d.ts +37 -0
- package/dist/components/FaceWarp3D/index.d.ts +3 -0
- package/dist/components/FaceWarp3D/warp.d.ts +13 -0
- package/dist/components/HandJoints2D/index.d.ts +6 -0
- package/dist/components/HandTracker/index.d.ts +2 -2
- package/dist/components/HandTracking2D/HandVisual.d.ts +12 -0
- package/dist/components/HandTracking2D/index.d.ts +5 -0
- package/dist/components/HandTracking2DController/index.d.ts +5 -0
- package/dist/components/HandTracking3D/index.d.ts +6 -0
- package/dist/components/LightObject/index.d.ts +20 -0
- package/dist/components/LookAt/index.d.ts +23 -0
- package/dist/components/MeshRenderer/index.d.ts +58 -0
- package/dist/components/PerspectiveCamera/index.d.ts +2 -8
- package/dist/components/PinToMesh/index.d.ts +47 -0
- package/dist/components/RenderGroupLayer/index.d.ts +6 -0
- package/dist/components/SceneObjectRenderer/index.d.ts +16 -0
- package/dist/components/Screen2D/ScreenTransformFrame.d.ts +11 -0
- package/dist/components/Screen2D/ScreenTransformHelpers.d.ts +19 -0
- package/dist/components/Screen2D/ScreenTransformInteraction.d.ts +18 -0
- package/dist/components/Screen2D/index.d.ts +5 -0
- package/dist/components/ScreenGlass/defaults.d.ts +3 -0
- package/dist/components/ScreenGlass/index.d.ts +5 -0
- package/dist/components/ScreenGlass/shaders.d.ts +2 -0
- package/dist/components/ScreenGlass/types.d.ts +10 -0
- package/dist/components/ScreenGradient/defaults.d.ts +5 -0
- package/dist/components/ScreenGradient/index.d.ts +5 -0
- package/dist/components/ScreenGradient/shaders.d.ts +2 -0
- package/dist/components/ScreenGradient/types.d.ts +19 -0
- package/dist/components/ScreenImage/ScreenImage.d.ts +4 -0
- package/dist/components/ScreenImage/components/DynamicTextureSync.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageBody.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageFrame.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageTransformFrame.d.ts +9 -0
- package/dist/components/ScreenImage/fit.d.ts +52 -0
- package/dist/components/ScreenImage/index.d.ts +8 -0
- package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts +9 -0
- package/dist/components/ScreenImage/materials/ScreenImageMatcap.d.ts +3 -0
- package/dist/components/ScreenImage/source.d.ts +16 -0
- package/dist/components/ScreenImage/types.d.ts +37 -0
- package/dist/components/ScreenImage/utils/ScreenImageConstants.d.ts +9 -0
- package/dist/components/ScreenImage/utils/ScreenImageHelpers.d.ts +4 -0
- package/dist/components/ScreenShape/geometry.d.ts +5 -0
- package/dist/components/ScreenShape/index.d.ts +5 -0
- package/dist/components/ScreenShape/types.d.ts +41 -0
- package/dist/components/ScreenSpaceUI/ScreenTransformGuide.d.ts +15 -0
- package/dist/components/ScreenSpaceUI/ScreenTransformOverlayGuide.d.ts +9 -0
- package/dist/components/ScreenSpaceUI/index.d.ts +2 -0
- package/dist/components/ScreenText/components/ScreenTextBody.d.ts +2 -0
- package/dist/components/ScreenText/components/ScreenTextTransformFrame.d.ts +9 -0
- package/dist/components/ScreenText/constants.d.ts +5 -0
- package/dist/components/ScreenText/index.d.ts +6 -0
- package/dist/components/ScreenText/types.d.ts +31 -0
- package/dist/components/ScreenText/utils/ScreenTextHelpers.d.ts +14 -0
- package/dist/components/ScreenText/utils/ScreenTextMappings.d.ts +17 -0
- package/dist/components/SequentialRenderPasses/index.d.ts +6 -0
- package/dist/components/TextLabel/TextLabel.d.ts +5 -0
- package/dist/components/TextLabel/alignment.d.ts +4 -0
- package/dist/components/TextLabel/bend.d.ts +9 -0
- package/dist/components/TextLabel/canvas/buildTextLabelTexture.d.ts +4 -0
- package/dist/components/TextLabel/canvas/canvasTypography.d.ts +13 -0
- package/dist/components/TextLabel/canvas/drawBackground.d.ts +21 -0
- package/dist/components/TextLabel/canvas/drawEffects.d.ts +34 -0
- package/dist/components/TextLabel/canvas/drawText.d.ts +19 -0
- package/dist/components/TextLabel/components/TextLabelMesh.d.ts +13 -0
- package/dist/components/TextLabel/constants.d.ts +2 -0
- package/dist/components/TextLabel/hooks/useTextLabelTexture.d.ts +7 -0
- package/dist/components/TextLabel/index.d.ts +2 -62
- package/dist/components/TextLabel/layout/measurement.d.ts +8 -0
- package/dist/components/TextLabel/layout/resolveTextLayout.d.ts +25 -0
- package/dist/components/TextLabel/layout/wrapText.d.ts +6 -0
- package/dist/components/TextLabel/types.d.ts +227 -0
- package/dist/components/TrackingAnchor/index.d.ts +18 -2
- package/dist/components/Transform3D/index.d.ts +30 -0
- package/dist/components/XRMediaSurface/index.d.ts +6 -6
- package/dist/components/XRScene/index.d.ts +14 -3
- package/dist/components/XRViewport/XRViewport.d.ts +11 -0
- package/dist/components/XRViewport/components/HiddenMedia.d.ts +7 -0
- package/dist/components/XRViewport/components/MediaAssetPicker.d.ts +8 -0
- package/dist/components/XRViewport/components/MediaControlStack.d.ts +13 -0
- package/dist/components/XRViewport/components/XRViewportLayers.d.ts +30 -0
- package/dist/components/XRViewport/constants.d.ts +24 -0
- package/dist/components/XRViewport/contexts.d.ts +6 -0
- package/dist/components/XRViewport/hooks/useViewportExternalFaces.d.ts +13 -0
- package/dist/components/XRViewport/hooks/useViewportMediaSource.d.ts +16 -0
- package/dist/components/XRViewport/hooks/useViewportMediaTexture.d.ts +10 -0
- package/dist/components/XRViewport/hooks/useViewportPipeline.d.ts +41 -0
- package/dist/components/XRViewport/hooks/useViewportRenderState.d.ts +47 -0
- package/dist/components/XRViewport/index.d.ts +3 -101
- package/dist/components/XRViewport/layout.d.ts +16 -0
- package/dist/components/XRViewport/slots.d.ts +16 -0
- package/dist/components/XRViewport/types.d.ts +89 -0
- package/dist/components/XRViewport/useViewportSize.d.ts +3 -0
- package/dist/components/index.d.ts +19 -0
- package/dist/contexts-BKvTIayr.cjs +1 -0
- package/dist/contexts-Bk5zz_VR.js +214 -0
- package/dist/debug-ui/component-editor/index.d.ts +19 -0
- package/dist/debug-ui/default-setting/index.d.ts +37 -0
- package/dist/debug-ui/default-setting.cjs +21 -0
- package/dist/debug-ui/default-setting.js +777 -0
- package/dist/debug-ui/face-mesh-renderer-settings/index.d.ts +8 -0
- package/dist/debug-ui/face-retouch/index.d.ts +11 -0
- package/dist/debug-ui/face-warp-3d/index.d.ts +6 -0
- package/dist/debug-ui/hand-joints-2d/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-2d/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-2d-controller/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-3d/index.d.ts +6 -0
- package/dist/debug-ui/hierarchy-panel/index.d.ts +29 -0
- package/dist/debug-ui/index.d.ts +46 -0
- package/dist/debug-ui/layer-panel/index.d.ts +18 -0
- package/dist/debug-ui/media-source/MediaSourceSwitcher.d.ts +10 -0
- package/dist/debug-ui/media-source/index.d.ts +6 -0
- package/dist/debug-ui/media-source/media-preset.d.ts +9 -0
- package/dist/debug-ui/media-source/session.d.ts +12 -0
- package/dist/debug-ui/media-source.cjs +1 -0
- package/dist/debug-ui/media-source.js +9 -0
- package/dist/debug-ui/mesh-renderer-settings/index.d.ts +6 -0
- package/dist/debug-ui/object-inspector-panel/index.d.ts +25 -0
- package/dist/debug-ui/pin-to-mesh-settings/index.d.ts +10 -0
- package/dist/debug-ui/scene-editor/context.d.ts +38 -0
- package/dist/debug-ui/scene-editor/hierarchy-panel.d.ts +9 -0
- package/dist/debug-ui/scene-editor/index.d.ts +11 -0
- package/dist/debug-ui/scene-editor/inspector-panel.d.ts +7 -0
- package/dist/debug-ui/scene-editor/store-utils.d.ts +2 -0
- package/dist/debug-ui/scene-editor/viewport-toolbar.d.ts +4 -0
- package/dist/debug-ui/scene-editor/viewport.d.ts +21 -0
- package/dist/debug-ui/screen-glass-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-gradient-settings/GradientStopsEditor.d.ts +5 -0
- package/dist/debug-ui/screen-gradient-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-gradient-settings/styles.d.ts +3 -0
- package/dist/debug-ui/screen-image-settings/index.d.ts +9 -0
- package/dist/debug-ui/screen-shape-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-text-settings/NumberField.d.ts +9 -0
- package/dist/debug-ui/screen-text-settings/ScreenTextSettingsPanel.d.ts +3 -0
- package/dist/debug-ui/screen-text-settings/controls/NestedSection.d.ts +7 -0
- package/dist/debug-ui/screen-text-settings/controls/Section.d.ts +8 -0
- package/dist/debug-ui/screen-text-settings/controls/icons.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/index.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/BackgroundSection.d.ts +8 -0
- package/dist/debug-ui/screen-text-settings/sections/ContentSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/EffectsSection.d.ts +25 -0
- package/dist/debug-ui/screen-text-settings/sections/StyleSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/TransformSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundFillControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundSizeControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundStrokeControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/types.d.ts +7 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/BendEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/EffectChrome.d.ts +21 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/GlowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/InnerShadowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/OutlineEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/ShadowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/types.d.ts +14 -0
- package/dist/debug-ui/screen-text-settings/settings.d.ts +90 -0
- package/dist/debug-ui/screen-text-settings/styles.d.ts +26 -0
- package/dist/debug-ui/screen-text-settings/transform.d.ts +6 -0
- package/dist/debug-ui/screen-text-settings/types.d.ts +9 -0
- package/dist/debug-ui/settings-panel/index.d.ts +38 -0
- package/dist/debug-ui/shared/MaterialSection.d.ts +13 -0
- package/dist/debug-ui/shared/ScreenTransformSection.d.ts +32 -0
- package/dist/debug-ui/transform-3d-settings/index.d.ts +11 -0
- package/dist/debug-ui.cjs +10 -0
- package/dist/debug-ui.js +8861 -0
- package/dist/drawNormalizedCrop-B-7zU-Xw.cjs +1 -0
- package/dist/drawNormalizedCrop-CRdIDyl1.js +44 -0
- package/dist/entries/all.d.ts +1 -0
- package/dist/entries/core.d.ts +55 -4
- package/dist/entries/face-effects.d.ts +12 -8
- package/dist/entries/internal.d.ts +1 -1
- package/dist/entries/low-level.d.ts +21 -4
- package/dist/entries/scene-object.d.ts +10 -0
- package/dist/entries/tracking.d.ts +25 -2
- package/dist/face-action/index.d.ts +34 -0
- package/dist/face-attribute-expression/index.d.ts +10 -0
- package/dist/face-effects.cjs +1 -252
- package/dist/face-effects.js +1404 -2613
- package/dist/face-info/blendshapes.d.ts +3 -0
- package/dist/face-info/derive.d.ts +8 -0
- package/dist/face-info/drawFaceCrop.d.ts +10 -0
- package/dist/face-info/getFaceInfo.d.ts +8 -0
- package/dist/face-info/index.d.ts +8 -0
- package/dist/face-info/landmarks.d.ts +21 -0
- package/dist/face-info/types.d.ts +52 -0
- package/dist/getFaceInfo-CyPAKh3h.cjs +1 -0
- package/dist/getFaceInfo-hUKMLFcr.js +118 -0
- package/dist/hand/handJoints2d.d.ts +49 -0
- package/dist/hand/handTracking2d.d.ts +33 -0
- package/dist/hand/handTracking2dController.d.ts +65 -0
- package/dist/hand/handTracking3d.d.ts +52 -0
- package/dist/hooks/faceResults.d.ts +1 -1
- package/dist/hooks/useCalibrationFaceTracker.d.ts +1 -1
- package/dist/hooks/useFaceAction.d.ts +3 -0
- package/dist/hooks/useFaceAttributeExpression.d.ts +16 -0
- package/dist/hooks/useFaceDetection.d.ts +18 -0
- package/dist/hooks/useFaceInfo.d.ts +7 -0
- package/dist/hooks/useFaceInfos.d.ts +15 -0
- package/dist/hooks/useFaceResultReader.d.ts +10 -0
- package/dist/hooks/useFrameCapture.d.ts +18 -0
- package/dist/hooks/useMediaRecorder.d.ts +14 -2
- package/dist/hooks/useOwnedTexture.d.ts +6 -0
- package/dist/hooks/useTrackerContext.d.ts +2 -2
- package/dist/hooks/useXRSession/facade.d.ts +3 -0
- package/dist/hooks/useXRSession/registry.d.ts +9 -0
- package/dist/index-82aENaRq.cjs +1 -0
- package/dist/{index-BD-JWMjE.js → index-BgQAX8rs.js} +198 -134
- package/dist/index-CEc1CAmB.cjs +1 -0
- package/dist/index-CTHeBQOs.js +178 -0
- package/dist/index-D4bLzBr6.cjs +1 -0
- package/dist/index-DSpzjuqd.cjs +1 -0
- package/dist/index-Di70KWuY.js +2049 -0
- package/dist/index-DzEzkBIH.cjs +17 -0
- package/dist/{index-CYMj26Ej.js → index-JygePOwo.js} +565 -498
- package/dist/index-UC0Ux38E.js +111 -0
- package/dist/{internal-KOKJDxCG.js → internal-DuMPpcbp.js} +86 -80
- package/dist/internal-i2JkM85N.cjs +1 -0
- package/dist/internal.cjs +1 -1
- package/dist/internal.js +2 -2
- package/dist/low-level.cjs +1 -1
- package/dist/low-level.js +959 -632
- package/dist/main.cjs +1 -1
- package/dist/main.js +556 -455
- package/dist/material/MaterialFill.d.ts +20 -0
- package/dist/material/MeshMaterial.d.ts +14 -0
- package/dist/material/index.d.ts +2 -0
- package/dist/models/Body/index.d.ts +1 -1
- package/dist/models/Face/index.d.ts +1 -0
- package/dist/models/Face/rotation.d.ts +3 -0
- package/dist/models/Hand/HandTracking.d.ts +2 -0
- package/dist/render/layerStack.d.ts +29 -0
- package/dist/render/renderGroups.d.ts +47 -0
- package/dist/render/renderPasses.d.ts +18 -0
- package/dist/renderGroups-C9QJrAXu.js +152 -0
- package/dist/renderGroups-Dvm1J4lW.cjs +1 -0
- package/dist/renderLayers-lDtL0_43.cjs +1 -0
- package/dist/renderLayers-zrK2pv5O.js +13 -0
- package/dist/renderPasses-CT5iBwtg.cjs +21 -0
- package/dist/renderPasses-CjgG_6kQ.js +1500 -0
- package/dist/scene-object/index.d.ts +146 -0
- package/dist/scene-object/runtimeRegistry.d.ts +14 -0
- package/dist/scene-object.cjs +1 -0
- package/dist/scene-object.js +40 -0
- package/dist/screen-image/screenImagePresets.d.ts +13 -0
- package/dist/screen-image/screenImageSettings.d.ts +57 -0
- package/dist/screen-text/screenTextPresets.d.ts +6 -0
- package/dist/screen-text/screenTextSettings.d.ts +147 -0
- package/dist/screen2d/coordinates.d.ts +87 -0
- package/dist/screen2d/index.d.ts +7 -0
- package/dist/screen2d/material-runtime.d.ts +32 -0
- package/dist/screen2d/screenTransform.d.ts +47 -0
- package/dist/screen2d/smoothing.d.ts +5 -0
- package/dist/screen2d/texture.d.ts +10 -0
- package/dist/screen2d/transform.d.ts +25 -0
- package/dist/screen2d/types.d.ts +90 -0
- package/dist/{selectors-Bub0o3qy.cjs → selectors-2QvO5RYY.cjs} +1 -1
- package/dist/{selectors-BHO6TnoC.js → selectors-Dw3C-gSk.js} +5 -5
- package/dist/services/MetricsCollector/index.d.ts +7 -1
- package/dist/services/MetricsCollector/utils.d.ts +1 -1
- package/dist/session-BChR4TgK.cjs +31 -0
- package/dist/session-C1StwN0f.js +363 -0
- package/dist/texture/animatedTexture.d.ts +61 -0
- package/dist/texture/animationMath.d.ts +8 -0
- package/dist/texture/sourceResolver.d.ts +30 -0
- package/dist/tracking.cjs +1 -1
- package/dist/tracking.js +582 -298
- package/dist/types/face.d.ts +1 -13
- package/dist/types/hand-tracking.d.ts +8 -0
- package/dist/types/index.d.ts +0 -1
- package/dist/types/nodes.d.ts +4 -1
- package/dist/types/session.d.ts +2 -0
- package/dist/types/tracking.d.ts +8 -2
- package/dist/types/util-types.d.ts +1 -1
- package/dist/useFaceInfo-C-WfMlYp.cjs +1 -0
- package/dist/useFaceInfo-CwsHZedC.js +62 -0
- package/dist/useFaceResults-CkFc1egZ.js +19 -0
- package/dist/useFaceResults-VcFdOUFV.cjs +1 -0
- package/dist/utils/drawNormalizedCrop.d.ts +22 -0
- package/dist/utils/integrity.d.ts +4 -25
- package/dist/utils/landmarkBounds.d.ts +1 -1
- package/dist/utils/logger.d.ts +2 -2
- package/dist/x-Co8jPhP0.js +127 -0
- package/dist/x-KC8E4Gaj.cjs +41 -0
- package/docs/MIGRATION_GUIDE.md +589 -0
- package/docs/SDK_DOCUMENTATION.md +1155 -0
- package/docs/api/audio.mdx +36 -0
- package/docs/api/behavior-components.mdx +164 -0
- package/docs/api/capture-recording.mdx +103 -0
- package/docs/api/debug-ui-scene-editor.mdx +249 -0
- package/docs/api/face-actions.mdx +104 -0
- package/docs/api/face-attribute-expression.mdx +87 -0
- package/docs/api/face-binding-anchors.mdx +53 -0
- package/docs/api/face-effects.mdx +19 -0
- package/docs/api/face-info.mdx +112 -0
- package/docs/api/face-mesh.mdx +98 -0
- package/docs/api/face-painting.mdx +33 -0
- package/docs/api/face-retouching.mdx +73 -0
- package/docs/api/material-render-state.mdx +132 -0
- package/docs/api/runtime-3d.mdx +311 -0
- package/docs/api/scene-object-store.mdx +213 -0
- package/docs/api/screen-image.mdx +41 -0
- package/docs/api/screen-shape.mdx +64 -0
- package/docs/api/screen-space-coordinates.mdx +41 -0
- package/docs/api/screen-text.mdx +39 -0
- package/docs/api/texture-sources.mdx +55 -0
- package/docs/api/trackers.mdx +46 -0
- package/docs/docs.json +70 -0
- package/docs/examples/scene-object-recipes.mdx +194 -0
- package/docs/guides/harness-debug-ui.mdx +139 -0
- package/docs/inspector/hierarchy-panel.mdx +51 -0
- package/docs/inspector/object-inspector-panel.mdx +51 -0
- package/docs/introduction.mdx +121 -0
- package/docs/scene-objects/overview.mdx +127 -0
- package/docs/scene-objects/render-groups.mdx +77 -0
- package/docs/scene-objects/runtime-rendering.mdx +75 -0
- package/docs/texture-api-consumer.md +58 -0
- package/package.json +55 -7
- package/dist/components/FaceRetouch/retouchRegions.d.ts +0 -23
- package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +0 -12
- package/dist/index-B5J4gzn2.cjs +0 -1
- package/dist/index-BVBVg96G.cjs +0 -1
- package/dist/index-DtwHElmG.js +0 -157
- package/dist/index-PtFmD6B8.cjs +0 -1
- package/dist/internal-DH5d1Xnq.cjs +0 -1
- package/dist/modules/XRClient/index.d.ts +0 -9
- package/dist/modules/index.d.ts +0 -1
- package/dist/services/ApiService.d.ts +0 -12
- package/dist/types/client.d.ts +0 -3
- package/dist/useFaceResults-cpsSmKKc.js +0 -81
- package/dist/useFaceResults-hVUJYSEY.cjs +0 -1
package/GROUNDING.md
CHANGED
|
@@ -10,7 +10,8 @@ or spelunk `node_modules` for the API.
|
|
|
10
10
|
The API is split into task-domain entry points. Import each symbol from the door
|
|
11
11
|
this reference shows for it; the imports here are literal.
|
|
12
12
|
|
|
13
|
-
- `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useMediaRecorder`, `dataURLtoFile`), audio.
|
|
13
|
+
- `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, runtime renderers (`Transform3D`, `MeshRenderer`, `SceneObjectRenderer`), `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useFrameCapture`, `useMediaRecorder`, `dataURLtoFile`), audio.
|
|
14
|
+
- `@vincentt-xr/sdk/scene-object` — **public authoring state**: scene-object/component types, factories, selectors, and immutable store operations.
|
|
14
15
|
- `@vincentt-xr/sdk/tracking` — **trackers**: face/hand/gesture/body/segmentation, `TrackingAnchor`, `GestureTrigger`, the bare `FaceMesh`, `useFaceResults`.
|
|
15
16
|
- `@vincentt-xr/sdk/face-effects` — **face deep**: FaceMesh material config, retouch, canonical mesh, head-binding.
|
|
16
17
|
- `@vincentt-xr/sdk/low-level` — **escape hatch**: raw model-node reads, selector hooks, custom-tracker plumbing. Reach here only when core + tracking can't express it.
|
|
@@ -24,6 +25,50 @@ logic is R3F `useFrame`; per-mount setup is `useEffect`.
|
|
|
24
25
|
> patterns) into the project's `GROUNDING.md`. Edit the SDK API here, in the repo
|
|
25
26
|
> that owns the API.
|
|
26
27
|
|
|
28
|
+
### Internal source layout
|
|
29
|
+
|
|
30
|
+
The public import doors above are stable. Internal files are organized by domain
|
|
31
|
+
so shared behavior stays reusable instead of duplicated in large components:
|
|
32
|
+
|
|
33
|
+
- `lib/components/ScreenImage/` — thin `ScreenImage.tsx` orchestrator plus
|
|
34
|
+
reusable body, frame, transform-frame, material, matcap, shader, source, and
|
|
35
|
+
helper modules.
|
|
36
|
+
- `lib/components/ScreenText/` — thin `ScreenText.tsx` orchestrator plus
|
|
37
|
+
reusable body, transform-frame, constants, mapping, and helper modules.
|
|
38
|
+
- `lib/components/TextLabel/` — low-level canvas text renderer split into
|
|
39
|
+
texture building, layout, style, hooks, and body modules.
|
|
40
|
+
- `lib/components/XRViewport/` — thin viewport orchestrator plus reusable
|
|
41
|
+
provider, stage, overlays, media, control, sizing, slot, and layout modules.
|
|
42
|
+
- `lib/components/Screen2D/` — shared screen-transform interaction/frame helpers
|
|
43
|
+
used by screen-space authoring components.
|
|
44
|
+
- `lib/debug-ui/screen-text-settings/` — panel orchestrator plus reusable
|
|
45
|
+
sections, controls, settings, styles, transform helpers, and types.
|
|
46
|
+
- `lib/render/` — render-group, render-pass, and layer-stack internals.
|
|
47
|
+
- `lib/hand/`, `lib/screen-image/`, and `lib/screen-text/` — domain-level
|
|
48
|
+
primitives, presets, settings, and tests that used to live flat in `lib/`.
|
|
49
|
+
|
|
50
|
+
When adding features, keep public exports at the documented import doors and put
|
|
51
|
+
implementation details in the matching domain folder. Do not reintroduce
|
|
52
|
+
component-local duplicate shader, transform, text-rendering, or settings logic.
|
|
53
|
+
|
|
54
|
+
### Example asset layout
|
|
55
|
+
|
|
56
|
+
Example-app assets are organized by purpose, not by whichever page first needed
|
|
57
|
+
them:
|
|
58
|
+
|
|
59
|
+
- `example/src/assets/media/images/` — reusable still textures and sample images.
|
|
60
|
+
- `example/src/assets/media/images/people/` — face/body tracking sample stills.
|
|
61
|
+
- `example/src/assets/media/images/screenshots/` — screenshot reference images.
|
|
62
|
+
- `example/src/assets/media/videos/` — sample MP4/WebM sources.
|
|
63
|
+
- `example/src/assets/media/gifs/` — animated GIF source files.
|
|
64
|
+
- `example/src/assets/media/sequences/<demo>/` — ordered image-sequence frames.
|
|
65
|
+
- `example/src/assets/models/` — GLB files used by example 3D scenes.
|
|
66
|
+
- `example/src/assets/face-effects/` — domain-owned face-effect assets.
|
|
67
|
+
|
|
68
|
+
Do not flatten all example assets into the root. Do not add new `utils/` media
|
|
69
|
+
folders. Compression is a separate visual/behavior review because it can change
|
|
70
|
+
tracking fixtures and texture output.
|
|
71
|
+
|
|
27
72
|
---
|
|
28
73
|
|
|
29
74
|
## Screen-space layout (the primary tool)
|
|
@@ -54,7 +99,9 @@ Coordinates are normalized: -1 to +1, with +y up. The center of the screen is `(
|
|
|
54
99
|
<mesh name="frame"><planeGeometry args={[1, 1]} /><meshBasicMaterial map={frameTex} transparent /></mesh>
|
|
55
100
|
</ScreenTransform>
|
|
56
101
|
|
|
57
|
-
// Bottom-center prompt
|
|
102
|
+
// Bottom-center prompt.
|
|
103
|
+
// Prefer <ScreenText> for ordinary screen copy; use this low-level pairing only
|
|
104
|
+
// when you need custom anchors.
|
|
58
105
|
<ScreenTransform anchors={{ left: -0.4, right: 0.4, top: -0.5, bottom: -0.9 }}>
|
|
59
106
|
<TextLabel name="prompt" text="Show your hand" fontSize={220} />
|
|
60
107
|
</ScreenTransform>
|
|
@@ -84,6 +131,196 @@ const [sx, sy] = computeContainScale(rectAspect, 1200 / 800); // content w/h rat
|
|
|
84
131
|
</mesh>;
|
|
85
132
|
```
|
|
86
133
|
|
|
134
|
+
### `<ScreenImage>` — Effect House-style screen-space image object
|
|
135
|
+
|
|
136
|
+
Use this for screen-space image planes such as logos, stickers, frames, badges,
|
|
137
|
+
and photo placeholders. `ScreenImage` owns its `ScreenTransform`; for normal
|
|
138
|
+
authoring prefer `position`, `size`, and `rotation`. `anchors` still exist as
|
|
139
|
+
the lower-level layout escape hatch. If no source is provided, it renders the
|
|
140
|
+
SDK-owned square default image.
|
|
141
|
+
|
|
142
|
+
### `<ScreenShape>` — editable vector primitives
|
|
143
|
+
|
|
144
|
+
Use `ScreenShape` for circles, ellipses, rectangles, rounded rectangles, regular
|
|
145
|
+
polygons, stars, and lines. Do not generate raster image assets for these basic
|
|
146
|
+
shapes. Placement and drag editing use the shared `ScreenTransform` settings.
|
|
147
|
+
Circles, regular polygons, and stars preserve equal axes inside non-square
|
|
148
|
+
transforms; ellipses and rectangles fill the transform.
|
|
149
|
+
|
|
150
|
+
```tsx
|
|
151
|
+
import { ScreenShape, ScreenSpaceUI } from "@vincentt-xr/sdk";
|
|
152
|
+
|
|
153
|
+
<ScreenSpaceUI>
|
|
154
|
+
<ScreenShape
|
|
155
|
+
transform={{ position: { x: 0, y: 0 }, size: { width: 220, height: 220 } }}
|
|
156
|
+
settings={{
|
|
157
|
+
kind: "star",
|
|
158
|
+
fillColor: "#ffd54a",
|
|
159
|
+
strokeColor: "#ffffff",
|
|
160
|
+
strokeWidth: 4,
|
|
161
|
+
sides: 5,
|
|
162
|
+
innerRadiusRatio: 0.45,
|
|
163
|
+
}}
|
|
164
|
+
/>
|
|
165
|
+
</ScreenSpaceUI>;
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Scene objects pair `ScreenTransform` with `ScreenShape`. Import scene-object
|
|
169
|
+
factories from `@vincentt-xr/sdk/scene-object` and the live
|
|
170
|
+
`ScreenShapeSettingsPanel` from `@vincentt-xr/sdk/debug-ui`.
|
|
171
|
+
|
|
172
|
+
Settings: `kind`; `fillMode`, `fillColor`, `fillOpacity`; `strokeColor`,
|
|
173
|
+
`strokeOpacity`, `strokeWidth`, `dashed`; `cornerRadius` for rounded rectangles;
|
|
174
|
+
`sides` for polygons/stars; `innerRadiusRatio` for stars; and normalized
|
|
175
|
+
`lineStart`/`lineEnd` `{ x, y }` coordinates for lines.
|
|
176
|
+
|
|
177
|
+
Set `fillMode: "gradient"` and provide `gradient: { type, stops, angle, center,
|
|
178
|
+
radius }`. Types are `linear`, `radial`, and `angular` (conic). A gradient accepts
|
|
179
|
+
2–8 `{ offset, color }` stops. `ScreenGradientMaterial` is a separate reusable
|
|
180
|
+
component exported from core; its settings panel is exported from debug-ui.
|
|
181
|
+
|
|
182
|
+
Set `fillMode: "glass"` and provide `glass: { tint, opacity, frost, highlight }`
|
|
183
|
+
for lightweight translucent glass. `ScreenGlassMaterial` is a separate reusable
|
|
184
|
+
component. It is a stylized overlay material, not backdrop blur/refraction.
|
|
185
|
+
|
|
186
|
+
### Shared material primitives
|
|
187
|
+
|
|
188
|
+
Import shared material primitives from `@vincentt-xr/sdk` only:
|
|
189
|
+
|
|
190
|
+
```tsx
|
|
191
|
+
import {
|
|
192
|
+
DEFAULT_MATERIAL_FILL,
|
|
193
|
+
MaterialFill,
|
|
194
|
+
MeshMaterial,
|
|
195
|
+
createTwoStopGradient,
|
|
196
|
+
type MaterialFillSettings,
|
|
197
|
+
type MeshMaterialSettings,
|
|
198
|
+
} from "@vincentt-xr/sdk";
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Use `MaterialFill` when a component needs the same solid/gradient/texture fill
|
|
202
|
+
system as `ScreenShape` and FaceMesh masks. `MaterialFillSettings.source` is
|
|
203
|
+
`"solid" | "gradient" | "texture"`; pass `color`, `opacity`, optional
|
|
204
|
+
`textureUrl`, and optional `gradient`.
|
|
205
|
+
|
|
206
|
+
```tsx
|
|
207
|
+
const fill: MaterialFillSettings = {
|
|
208
|
+
...DEFAULT_MATERIAL_FILL,
|
|
209
|
+
source: "gradient",
|
|
210
|
+
color: "#38bdf8",
|
|
211
|
+
opacity: 0.8,
|
|
212
|
+
gradient: createTwoStopGradient("#38bdf8"),
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
<mesh>
|
|
216
|
+
<planeGeometry args={[1, 1]} />
|
|
217
|
+
<MaterialFill settings={fill} />
|
|
218
|
+
</mesh>;
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Use `MeshMaterial` for serialized mesh material settings shared by
|
|
222
|
+
`MeshRenderer`; do not re-create local `<meshBasicMaterial>` /
|
|
223
|
+
`<meshStandardMaterial>` branches in scene-object components.
|
|
224
|
+
|
|
225
|
+
```tsx
|
|
226
|
+
import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
|
|
227
|
+
|
|
228
|
+
<ScreenSpaceUI>
|
|
229
|
+
<ScreenImage
|
|
230
|
+
name="defaultSquareImage"
|
|
231
|
+
anchors={{ left: -0.35, right: 0.35, top: 0.35, bottom: -0.35 }}
|
|
232
|
+
fit="contain"
|
|
233
|
+
/>
|
|
234
|
+
</ScreenSpaceUI>
|
|
235
|
+
|
|
236
|
+
<ScreenSpaceUI>
|
|
237
|
+
<ScreenImage
|
|
238
|
+
name="logo"
|
|
239
|
+
source={{ kind: "image", src: "https://cdn.../logo.png" }}
|
|
240
|
+
coordinateSpace="canvas"
|
|
241
|
+
canvasSize={{ width: 720, height: 1280 }}
|
|
242
|
+
position={[-180, 464]}
|
|
243
|
+
size={[216, 160]}
|
|
244
|
+
stretchMode="fit"
|
|
245
|
+
opacity={0.95}
|
|
246
|
+
/>
|
|
247
|
+
</ScreenSpaceUI>
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Props:
|
|
251
|
+
|
|
252
|
+
- `source`: `{ kind: "default" }`, `{ kind: "image", src }`, or `{ kind: "texture", texture }`.
|
|
253
|
+
- `src`: shorthand for `{ kind: "image", src }`.
|
|
254
|
+
- `fit`: `"stretch" | "contain" | "cover"`; default `"contain"`.
|
|
255
|
+
- `coordinateSpace`: `"normalized" | "canvas"`; default `"normalized"`.
|
|
256
|
+
- `canvasSize`: design canvas size for `coordinateSpace="canvas"`; default
|
|
257
|
+
`{ width: 720, height: 1280 }`.
|
|
258
|
+
- `position`, `size`, `rotation`: author-facing screen-object transform.
|
|
259
|
+
In normalized mode, `position` is the rect center in normalized screen space
|
|
260
|
+
and `size` is normalized rect width/height. In canvas mode, `position` and
|
|
261
|
+
`size` are design-canvas pixels with center origin and +y up, matching the
|
|
262
|
+
Effect House 720x1280 Screen Transform convention.
|
|
263
|
+
- `anchors`, `offsets`, `pivot`: forwarded to the owned `ScreenTransform`.
|
|
264
|
+
- `opacity`, `color`, `flipX`, `flipY`, `renderOrder`, `transparent`, `alphaTest`, `overlay`, `visible`.
|
|
265
|
+
|
|
266
|
+
Effect House transfer rule: Screen Transform `Size` is the screen-space
|
|
267
|
+
bounding rectangle, not the image's natural pixel size. The image then fits
|
|
268
|
+
inside that rectangle according to `stretchMode`. Fit/fill/stretch changes the
|
|
269
|
+
texture content inside the rectangle; it must not change the object's rigid
|
|
270
|
+
transform. Rotation rotates the whole rectangle around `pivot`.
|
|
271
|
+
|
|
272
|
+
Settings:
|
|
273
|
+
|
|
274
|
+
- `DEFAULT_SCREEN_IMAGE_SETTINGS.transformation`: coordinate space, canvas size,
|
|
275
|
+
position, size, rotation, pivot, render order, overlay/depth behavior, and visibility.
|
|
276
|
+
- `DEFAULT_SCREEN_IMAGE_SETTINGS.image`: source kind, src, fit, opacity, color,
|
|
277
|
+
flips, transparency, and alpha test.
|
|
278
|
+
- The component source union is intentionally small: `default`, static `image`,
|
|
279
|
+
and an already-loaded Three `texture`. Video/GIF/sequence providers resolve
|
|
280
|
+
their media to a live `THREE.Texture` and pass it as `kind: "texture"`.
|
|
281
|
+
|
|
282
|
+
Use `<ScreenImage>` instead of hand-authoring `<ScreenTransform>` + `<mesh>`
|
|
283
|
+
when the object is conceptually a 2D screen image. Use the lower-level mesh path
|
|
284
|
+
only when you need custom geometry or material behavior.
|
|
285
|
+
|
|
286
|
+
### Shared debug authoring surface for `ScreenImage`
|
|
287
|
+
|
|
288
|
+
The SDK ships a dev-only inspector at `lib/debug-ui/screen-image-settings` for
|
|
289
|
+
authoring `ScreenImage` settings in creator tooling and the v2 template
|
|
290
|
+
preview. It exposes the image renderer, transform, material, and fit controls
|
|
291
|
+
as one shared panel. End-user shipping surfaces should not include the debug
|
|
292
|
+
chrome.
|
|
293
|
+
|
|
294
|
+
The panel's transform copy button copies a JSON payload for the selected object
|
|
295
|
+
including `name`, `kind: "screenImage"`, `transform`, `image`, `material`, and
|
|
296
|
+
`screenSpaceUI`. The `transform` includes `coordinateSpace` and `canvasSize`.
|
|
297
|
+
Use that payload as the handoff from human layout tuning back to code.
|
|
298
|
+
|
|
299
|
+
### ScreenImage contract for every caller
|
|
300
|
+
|
|
301
|
+
`ScreenImage` is the canonical screen-image object in SDK and client apps. Any
|
|
302
|
+
debug panel, hierarchy row, editor, or runtime controller that targets one must
|
|
303
|
+
operate on the same mounted object state:
|
|
304
|
+
|
|
305
|
+
- Keep one source of truth for `transform`, source/material settings, and
|
|
306
|
+
visibility. Panel edits must immediately update the rendered object.
|
|
307
|
+
- Always wire `onScreenTransformChange` for inspector or editor usage. It is the
|
|
308
|
+
real drag/place path; do not implement a panel-only shadow transform.
|
|
309
|
+
- Use `coordinateSpace="canvas"` plus `canvasSize` for authoring tools, then
|
|
310
|
+
map canvas values to `position`/`size`; never treat design pixels as world
|
|
311
|
+
units. `fit`/`stretchMode` changes content fitting, not the object's bounds.
|
|
312
|
+
- Source replacement must preserve the object identity when possible, stop and
|
|
313
|
+
dispose the previous owned provider, and keep the current transform/material
|
|
314
|
+
settings. Dynamic video/canvas textures must continue marking fitted clones
|
|
315
|
+
for GPU upload.
|
|
316
|
+
- Respect capability-gated controls: static images expose image/material
|
|
317
|
+
controls only; animated providers expose only controls they support. Loading,
|
|
318
|
+
error, retry, pause, ended, and cleanup state must be visible to diagnostics.
|
|
319
|
+
- `visible`, `overlay`, `renderOrder`, `transparent`, `alphaTest`, blend mode,
|
|
320
|
+
opacity, tint, flips, pivot, rotation, size, and position are all part of the
|
|
321
|
+
live object contract and must remain synchronized from SDK or client entry
|
|
322
|
+
points.
|
|
323
|
+
|
|
87
324
|
### `<SpriteAnimation sheet={{ url, cols, rows, frames, fps }}>` — Animated transparent overlay
|
|
88
325
|
|
|
89
326
|
The primitive for an animated transparent overlay (a frame border, a gesture-instruction graphic, an animated sticker). One transparent WebP atlas of `cols`×`rows` cells; the first `frames` play left-to-right, top-to-bottom at `fps`. It is a single static GPU texture with a per-frame UV offset — no video decode — so it works in every native and in-app browser and costs ~zero heat.
|
|
@@ -115,9 +352,223 @@ import { SpriteAnimation, ScreenSpaceUI } from "@vincentt-xr/sdk";
|
|
|
115
352
|
|
|
116
353
|
Mount the tracker component; it registers the underlying ML model automatically. No `registerXRPipeline` call needed.
|
|
117
354
|
|
|
355
|
+
### 3D face warp — FBX head binding and face proportion deformation
|
|
356
|
+
|
|
357
|
+
Use `faceWarp3D` on `<CanonicalFaceMeshRenderer>` to deform the live canonical
|
|
358
|
+
face mesh before projection. Defaults are no-op; pass `FACE_WARP_3D_PRESET` for
|
|
359
|
+
a visible starter look, or provide partial settings.
|
|
360
|
+
|
|
361
|
+
```tsx
|
|
362
|
+
import { CanonicalFaceMeshRenderer, FACE_WARP_3D_PRESET } from "@vincentt-xr/sdk/face-effects";
|
|
363
|
+
|
|
364
|
+
<CanonicalFaceMeshRenderer
|
|
365
|
+
materialType="shader"
|
|
366
|
+
shaderPreset="originalFace"
|
|
367
|
+
textureMap="none"
|
|
368
|
+
alphaMap="canonical"
|
|
369
|
+
alphaOpacity={1}
|
|
370
|
+
baseColor="#ffffff"
|
|
371
|
+
opacity={1}
|
|
372
|
+
side="front"
|
|
373
|
+
faceAlphaMaskUrl="https://cdn.vincentt.studio/assets/sdk/face/v1/Canonical_Face_Color_Mask.png"
|
|
374
|
+
faceWarp3D={FACE_WARP_3D_PRESET}
|
|
375
|
+
/>;
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
Settings (`FaceWarp3DSettings`) include `faceWidth`, `faceHeight`,
|
|
379
|
+
`foreheadHeight`, `cheekWidth`, `eyeSize`, `eyeDistance`, `noseSize`,
|
|
380
|
+
`noseLength`, `mouthWidth`, `mouthHeight`, `jawWidth`, `chinHeight`, plus
|
|
381
|
+
`enabled` and `strength`. Setting values are normalized around zero; typical UI
|
|
382
|
+
sliders map `-100..100` to `-1..1`.
|
|
383
|
+
|
|
384
|
+
For local tuning, import `FaceWarp3DSettingsPanel` from
|
|
385
|
+
`@vincentt-xr/sdk/debug-ui`. It is a draggable debug panel and should not
|
|
386
|
+
be included in the final production scene.
|
|
387
|
+
|
|
388
|
+
### `FaceMeshRenderer` — scene-object live 3D face mesh
|
|
389
|
+
|
|
390
|
+
Use `FaceMeshRenderer` for the standard SDK face-mesh scene-object component.
|
|
391
|
+
It owns face tracking internally through `FaceTracker`, reads live landmarks,
|
|
392
|
+
deforms the canonical face mesh, and renders through normal face-mesh material
|
|
393
|
+
settings. Prefer this over direct `CanonicalFaceMeshRenderer` in new
|
|
394
|
+
scene-object/editor integrations.
|
|
395
|
+
|
|
396
|
+
Import runtime/material symbols from `@vincentt-xr/sdk/face-effects`:
|
|
397
|
+
|
|
398
|
+
```tsx
|
|
399
|
+
import {
|
|
400
|
+
DEFAULT_FACE_MESH_RENDERER,
|
|
401
|
+
DEFAULT_FACE_MESH_TRANSFORM,
|
|
402
|
+
FACE_MESH_MATERIAL_TYPES,
|
|
403
|
+
FACE_MESH_SIDES,
|
|
404
|
+
FaceMeshRenderer,
|
|
405
|
+
mapFaceMeshRendererSettingsToProps,
|
|
406
|
+
resolveFaceMeshRendererSettings,
|
|
407
|
+
type FaceMeshRendererSettings,
|
|
408
|
+
} from "@vincentt-xr/sdk/face-effects";
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
Import scene-object defaults and store helpers from
|
|
412
|
+
`@vincentt-xr/sdk/scene-object`:
|
|
413
|
+
|
|
414
|
+
```tsx
|
|
415
|
+
import {
|
|
416
|
+
createDefaultComponent,
|
|
417
|
+
createDefaultFaceMeshRendererSettings,
|
|
418
|
+
DEFAULT_RENDER_GROUP_IDS,
|
|
419
|
+
type RenderGroupSceneObjectStore,
|
|
420
|
+
} from "@vincentt-xr/sdk/scene-object";
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
`FaceMeshRendererSettings` is the serialized component settings contract:
|
|
424
|
+
`enabled`, `faceIndex`, `targetFps`, `showLeftEyeGeometry`,
|
|
425
|
+
`showRightEyeGeometry`, `showMouthGeometry`, `showNoseGeometry`,
|
|
426
|
+
`showOverlayMask`, `showMouthMask`, `showLeftEyeMask`, `showRightEyeMask`,
|
|
427
|
+
`showNoseMask`, `externalMeshUrl`, `externalScale`, `materialType`,
|
|
428
|
+
`shaderPreset`, `textureMap`, `alphaMap`, `alphaOpacity`, `side`, `blendMode`,
|
|
429
|
+
`blendOpacity`, `baseColor`, `opacity`, `compositeShading`,
|
|
430
|
+
`maskTextureSource`, `maskSourceUrl`, `maskColor`, `maskOpacity`,
|
|
431
|
+
`faceTextureUrl`, `faceTextureStyle2Url`, `faceAlphaMaskUrl`, and
|
|
432
|
+
`renderOrder`.
|
|
433
|
+
|
|
434
|
+
`maskTextureSource` is `"solid" | "gradient" | "imageUrl" | "mediaUrl"`.
|
|
435
|
+
Solid and gradient masks use `maskColor`; image/media masks use
|
|
436
|
+
`maskSourceUrl`. `maskOpacity` is always `0..1`.
|
|
437
|
+
|
|
438
|
+
`materialType: "shader"` has three shader presets: `"basicShader"`,
|
|
439
|
+
`"originalFace"`, and `"composite"`. Use
|
|
440
|
+
`resolveFaceMeshRendererSettings(current, patch)` when editing settings so
|
|
441
|
+
shader/material/texture state stays synchronized. Selecting composite applies
|
|
442
|
+
the SDK composite defaults (`textureMap: "canonical"`, hard-light blend,
|
|
443
|
+
`blendOpacity: 0.85`, `compositeShading: true`) instead of silently falling
|
|
444
|
+
back to original-face rendering.
|
|
445
|
+
|
|
446
|
+
`externalMeshUrl` is for canonical-compatible OBJ assets that match the
|
|
447
|
+
renderer's live landmark deformation path. Third-party Face Mesh OBJ files are
|
|
448
|
+
reference assets only until the SDK has an explicit topology mapper from those
|
|
449
|
+
vertices to the 468 tracked face landmarks. Do not bundle OBJ/FBX/PNG reference
|
|
450
|
+
files in `lib/` or examples unless runtime code consumes them correctly.
|
|
451
|
+
|
|
452
|
+
Do not add settings that are not runtime-effective. Skull/ear geometry,
|
|
453
|
+
expression multiplier, and UV1 mapping require parser/runtime support before
|
|
454
|
+
they become public SDK settings.
|
|
455
|
+
|
|
456
|
+
Scene-object example:
|
|
457
|
+
|
|
458
|
+
```tsx
|
|
459
|
+
const objectId = "face-mesh";
|
|
460
|
+
const transform = createDefaultComponent("Transform", objectId);
|
|
461
|
+
transform.settings.position.y = DEFAULT_FACE_MESH_TRANSFORM.position.y;
|
|
462
|
+
|
|
463
|
+
const faceMesh = createDefaultComponent("FaceMeshRenderer", objectId);
|
|
464
|
+
faceMesh.settings = {
|
|
465
|
+
...createDefaultFaceMeshRendererSettings(),
|
|
466
|
+
materialType: FACE_MESH_MATERIAL_TYPES.wireframe,
|
|
467
|
+
side: FACE_MESH_SIDES.double,
|
|
468
|
+
baseColor: "#00e5ff",
|
|
469
|
+
opacity: 0.82,
|
|
470
|
+
} satisfies FaceMeshRendererSettings;
|
|
471
|
+
|
|
472
|
+
const store: RenderGroupSceneObjectStore = {
|
|
473
|
+
objects: {
|
|
474
|
+
[objectId]: {
|
|
475
|
+
id: objectId,
|
|
476
|
+
name: "Face Mesh",
|
|
477
|
+
enabled: true,
|
|
478
|
+
renderGroupId: DEFAULT_RENDER_GROUP_IDS.scene3d,
|
|
479
|
+
componentIds: [transform.id, faceMesh.id],
|
|
480
|
+
},
|
|
481
|
+
},
|
|
482
|
+
components: {
|
|
483
|
+
[transform.id]: transform,
|
|
484
|
+
[faceMesh.id]: faceMesh,
|
|
485
|
+
},
|
|
486
|
+
renderGroups: {
|
|
487
|
+
[DEFAULT_RENDER_GROUP_IDS.scene3d]: {
|
|
488
|
+
id: DEFAULT_RENDER_GROUP_IDS.scene3d,
|
|
489
|
+
name: "3D Scene",
|
|
490
|
+
domain: "scene3d",
|
|
491
|
+
enabled: true,
|
|
492
|
+
objectIds: [objectId],
|
|
493
|
+
},
|
|
494
|
+
},
|
|
495
|
+
renderGroupOrder: [DEFAULT_RENDER_GROUP_IDS.scene3d],
|
|
496
|
+
};
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
When `SceneObjectRenderer` sees both `Transform` and `FaceMeshRenderer` on the
|
|
500
|
+
same object, it passes the authored `Transform` into `FaceMeshRenderer`.
|
|
501
|
+
FaceMesh does not use normal world-space `Transform3D` semantics. It applies
|
|
502
|
+
the supported Transform values in MediaPipe landmark space before camera
|
|
503
|
+
projection; it must never wrap the already-projected mesh in a parent
|
|
504
|
+
`Transform3D`, because that causes depth-dependent screen skew.
|
|
505
|
+
|
|
506
|
+
Supported FaceMesh Transform values are position and uniform scale only.
|
|
507
|
+
Rotation and non-uniform scale are ignored by the runtime until the SDK has a
|
|
508
|
+
real object-space face solve. Authored `position.y` is Y-up, but MediaPipe image
|
|
509
|
+
space is Y-down, so the renderer subtracts Y while adding X/Z.
|
|
510
|
+
|
|
511
|
+
This is the SDK's strict FaceMesh contract: FaceInfo/FaceBinding pose first,
|
|
512
|
+
FaceMesh deformation in face landmark space, supported authored Transform in
|
|
513
|
+
that space, then world/camera projection.
|
|
514
|
+
`DEFAULT_FACE_MESH_TRANSFORM` starts from `DEFAULT_TRANSFORM_3D` and sets
|
|
515
|
+
`position.y` to `0.001` as an intentional FaceMesh authoring bias, not as a
|
|
516
|
+
projection correction. Author a Transform with `position.y = 0` to opt out.
|
|
517
|
+
|
|
518
|
+
`mapFaceMeshRendererSettingsToProps(settings)` maps serialized
|
|
519
|
+
`FaceMeshRendererSettings` to low-level `CanonicalFaceMeshRendererProps`.
|
|
520
|
+
Use it only when bridging custom runtime code; ordinary scene-object rendering
|
|
521
|
+
should use `SceneObjectRenderer`.
|
|
522
|
+
|
|
523
|
+
`CanonicalFaceMeshRenderer` remains exported from `@vincentt-xr/sdk/face-effects`
|
|
524
|
+
for backwards compatibility and low-level experiments, but new editor and
|
|
525
|
+
scene-object work should use `FaceMeshRenderer`.
|
|
526
|
+
|
|
527
|
+
### `FaceRetouch` — legacy landmark skin retouch
|
|
528
|
+
|
|
529
|
+
Use `FaceRetouch` from `@vincentt-xr/sdk/face-effects` for the legacy
|
|
530
|
+
landmark-masked skin retouch shader. The public controls are intentionally
|
|
531
|
+
minimal:
|
|
532
|
+
|
|
533
|
+
```tsx
|
|
534
|
+
import {
|
|
535
|
+
FaceRetouch,
|
|
536
|
+
patchFaceRetouchControls,
|
|
537
|
+
resolveFaceRetouchControls,
|
|
538
|
+
type FaceRetouchControls,
|
|
539
|
+
} from "@vincentt-xr/sdk/face-effects";
|
|
540
|
+
import { FaceRetouchSettingsPanel } from "@vincentt-xr/sdk/debug-ui";
|
|
541
|
+
|
|
542
|
+
const [controls, setControls] = useState(() => resolveFaceRetouchControls());
|
|
543
|
+
|
|
544
|
+
const updateControls = (next: Partial<FaceRetouchControls>) => {
|
|
545
|
+
setControls((current) => patchFaceRetouchControls(current, next));
|
|
546
|
+
};
|
|
547
|
+
|
|
548
|
+
<FaceRetouch {...controls} />;
|
|
549
|
+
<FaceRetouchSettingsPanel value={controls} onChange={updateControls} />;
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
`FaceRetouchControls` has only `skinSmoothing`, `skinRealism`,
|
|
553
|
+
`blemishReduction`, and `previewMask`. Do not invent provider, quality,
|
|
554
|
+
redness, teeth, eye, or blur controls for this component.
|
|
555
|
+
|
|
556
|
+
For multiple people, render one `FaceRetouch` per face:
|
|
557
|
+
|
|
558
|
+
```tsx
|
|
559
|
+
<FaceRetouch {...controls} faceIndex={0} numFaces={2} />
|
|
560
|
+
<FaceRetouch {...controls} faceIndex={1} numFaces={2} />
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
`numFaces` defaults to at least `faceIndex + 1`. `clipMinX` and `clipMaxX` are
|
|
564
|
+
optional horizontal UV clipping props in `0..1` space, useful for split-screen
|
|
565
|
+
comparison demos.
|
|
566
|
+
|
|
118
567
|
### `<FaceTracker>` — Face tracking (468 landmarks)
|
|
119
568
|
|
|
120
|
-
Detects faces and provides face transform to children via context. Supports
|
|
569
|
+
Detects faces and provides face transform to children via context. Supports
|
|
570
|
+
`faceIndex` (default `0`), `scaleWithFace`, and `followFaceRoll` (default
|
|
571
|
+
`true`; set `false` to keep children upright while the head tilts).
|
|
121
572
|
|
|
122
573
|
```tsx
|
|
123
574
|
import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
|
|
@@ -132,6 +583,274 @@ import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
|
|
|
132
583
|
</FaceTracker>;
|
|
133
584
|
```
|
|
134
585
|
|
|
586
|
+
### `FaceInfo` — face-driven effect logic
|
|
587
|
+
|
|
588
|
+
Use `FaceInfo` for head-pose, expression, landmark, bounds, and face-scale logic.
|
|
589
|
+
Do not read raw MediaPipe results or repeat landmark indices unless the facade
|
|
590
|
+
does not expose data required by a research/debugging tool.
|
|
591
|
+
|
|
592
|
+
```tsx
|
|
593
|
+
import { hasFaceExpression, useFaceInfo } from "@vincentt-xr/sdk/tracking";
|
|
594
|
+
|
|
595
|
+
function FaceLogic() {
|
|
596
|
+
const face = useFaceInfo({ active: true });
|
|
597
|
+
if (!face) return null;
|
|
598
|
+
|
|
599
|
+
const mouthOpen = hasFaceExpression(face, "jawOpen");
|
|
600
|
+
const lookingLeft = face.pose.rotation.y > 0.2;
|
|
601
|
+
return <group visible={mouthOpen} rotation-y={lookingLeft ? 0.2 : 0} />;
|
|
602
|
+
}
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
| Field | Coordinate contract |
|
|
606
|
+
| --------------- | ---------------------------------------------------------------- |
|
|
607
|
+
| `index` | Current result-array index; not a persistent identity |
|
|
608
|
+
| `landmarks` | MediaPipe normalized tracker-input coordinates, top-left origin |
|
|
609
|
+
| `bounds` | Normalized tracker-input bounds, top-left origin |
|
|
610
|
+
| `pose.position` | Existing face-tracker Three.js world position |
|
|
611
|
+
| `pose.rotation` | Radians: X pitch (+up), Y yaw (+left), Z roll |
|
|
612
|
+
| `pose.scale` | Smoothed uniform sizing heuristic used by face anchors/occluders |
|
|
613
|
+
| `eyeDistance` | Outer-eye distance in normalized tracker-input coordinates |
|
|
614
|
+
| `blendshapes` | MediaPipe category names and scores in `[0, 1]` |
|
|
615
|
+
|
|
616
|
+
`useFaceInfo({ faceIndex })` is the normal single-face React reader.
|
|
617
|
+
`useFaceInfos({ numFaces })` publishes all current faces. Both acquire the shared
|
|
618
|
+
face model through the SDK registry and use the existing throttle/empty-frame
|
|
619
|
+
hold lifecycle; do not mount a separate result polling loop. Call them inside
|
|
620
|
+
`XRProvider` and the R3F Canvas. Options are `active`, `numFaces`, `targetFps`,
|
|
621
|
+
`throttleMs`, and `holdMs`; `useFaceInfo` additionally accepts `faceIndex`.
|
|
622
|
+
|
|
623
|
+
```tsx
|
|
624
|
+
function MultiFaceLogic() {
|
|
625
|
+
const faces = useFaceInfos({ numFaces: 3 });
|
|
626
|
+
return <group visible={faces.some((face) => hasFaceExpression(face, "jawOpen"))} />;
|
|
627
|
+
}
|
|
628
|
+
```
|
|
629
|
+
|
|
630
|
+
For `useFrame` code that already has a low-level face node, call
|
|
631
|
+
`getFaceInfo(node, faceIndex, target)`. Create `target` once with
|
|
632
|
+
`createFaceInfoTarget()` and reuse it to avoid per-frame allocations. Use
|
|
633
|
+
`getFaceInfo` and `createFaceInfoTarget` from `@vincentt-xr/sdk/low-level`; they
|
|
634
|
+
are node-in escape hatches, not the normal React API. Use `getFaceBlendshape`
|
|
635
|
+
and `hasFaceExpression`; absent scores resolve to `0`, and the default expression
|
|
636
|
+
threshold is `DEFAULT_FACE_EXPRESSION_THRESHOLD`. Bind the R3F camera once with
|
|
637
|
+
`node.setCamera(camera, faceIndex)` before reading pose; the React hooks do this
|
|
638
|
+
automatically.
|
|
639
|
+
|
|
640
|
+
All supported expression names are typed by `FaceBlendshapeName` and listed in
|
|
641
|
+
the greppable `FACE_BLENDSHAPE_NAMES` constant exported from
|
|
642
|
+
`@vincentt-xr/sdk/tracking`. This includes `eyeBlinkLeft`, `eyeBlinkRight`,
|
|
643
|
+
`eyeWideLeft`, `eyeWideRight`, and `jawOpen`; do not guess string names.
|
|
644
|
+
|
|
645
|
+
Use `normalizedToScreenPixels` with the tracker input/display viewport size to
|
|
646
|
+
display a face landmark in Screen Image/Text/Shape coordinates. The tracker
|
|
647
|
+
input is already the SDK feed; do not apply media cover/mirroring a second time.
|
|
648
|
+
Never mutate or mirror `FaceInfo.landmarks` in place, manually project them,
|
|
649
|
+
infer a persistent face ID, or copy numeric landmark indices/thresholds into
|
|
650
|
+
application code.
|
|
651
|
+
|
|
652
|
+
```ts
|
|
653
|
+
const nose = normalizedToScreenPixels({
|
|
654
|
+
point: face.landmarks[FACE_LANDMARK_INDICES["face.noseTip"]],
|
|
655
|
+
viewportSize,
|
|
656
|
+
});
|
|
657
|
+
```
|
|
658
|
+
|
|
659
|
+
Effect House mapping: `FaceInfo.landmarks` replaces `pointsArray`, `bounds`
|
|
660
|
+
replaces `rect`, `pose.rotation` provides pitch/yaw/roll, and `eyeDistance` is
|
|
661
|
+
named the same. Use Face Actions below for the semantic `FaceAction` role.
|
|
662
|
+
Vincentt uses MediaPipe's face topology and top-left origin; it is not a
|
|
663
|
+
106-point or bottom-left-compatible alias.
|
|
664
|
+
|
|
665
|
+
Run the example route `/face-info` to verify all landmarks, live bounds, nose
|
|
666
|
+
position, pose, eye distance, and `jawOpen` using `ScreenShape`, `ScreenText`,
|
|
667
|
+
and `Axes`.
|
|
668
|
+
|
|
669
|
+
### `useFaceDetection()` — face presence lifecycle and count
|
|
670
|
+
|
|
671
|
+
Import `useFaceDetection` from `@vincentt-xr/sdk/tracking` when logic needs face
|
|
672
|
+
detected/lost edges, selected-face presence, or the current face count. It wraps
|
|
673
|
+
the shared `useFaceInfos` reader and does not start another model.
|
|
674
|
+
|
|
675
|
+
```tsx
|
|
676
|
+
const detection = useFaceDetection({ faceIndex: 0, numFaces: 5 });
|
|
677
|
+
|
|
678
|
+
if (detection.justDetected) showEffect();
|
|
679
|
+
if (detection.detected) updateEffect(detection.face);
|
|
680
|
+
if (detection.justLost) hideEffect();
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
| Visual-scripting term | SDK field |
|
|
684
|
+
| ------------------------------------- | ------------------------------------ |
|
|
685
|
+
| Face Detection Begin / face appeared | `justDetected`, `phase === "begin"` |
|
|
686
|
+
| Face Detection Stay / face present | `detected`, `phase === "stay"` |
|
|
687
|
+
| Face Detection End / face disappeared | `justLost`, `phase === "end"` |
|
|
688
|
+
| Face Detection None / no face | `!detected`, `phase === "none"` |
|
|
689
|
+
| Face Count / number of faces | `faceCount` and `faces.length` |
|
|
690
|
+
| Which Face / selected face | `faceIndex` option and `face` result |
|
|
691
|
+
|
|
692
|
+
The full result is `face`, `faces`, `faceCount`, `detected`, `justDetected`,
|
|
693
|
+
`justLost`, and `phase`. Options are the normal `useFaceInfos` options plus
|
|
694
|
+
`faceIndex`: `active`, `numFaces`, `targetFps`, `throttleMs`, and `holdMs`.
|
|
695
|
+
Default `holdMs: 250` prevents a transient missed frame from firing `end`; use
|
|
696
|
+
`holdMs: 0` only when immediate disappearance is required. Do not compare raw
|
|
697
|
+
result arrays or build a local previous-frame ref. The verified
|
|
698
|
+
`/face-detection` page tests five selectable face slots and all four phases; its
|
|
699
|
+
real-camera lifecycle flow passed on 26 July 2026.
|
|
700
|
+
|
|
701
|
+
### Reusable normalized and face image crops
|
|
702
|
+
|
|
703
|
+
Import `drawNormalizedCrop` from `@vincentt-xr/sdk` for any normalized image
|
|
704
|
+
region. Import `drawFaceCrop` from `@vincentt-xr/sdk/tracking` when the bounds
|
|
705
|
+
come from `FaceInfo`. Both preserve the requested output pixel aspect ratio,
|
|
706
|
+
clamp at source edges, and draw into a caller-owned canvas without starting a
|
|
707
|
+
tracker or allocating per call.
|
|
708
|
+
|
|
709
|
+
```ts
|
|
710
|
+
const cropCanvas = document.createElement("canvas");
|
|
711
|
+
|
|
712
|
+
const drawn = drawFaceCrop({
|
|
713
|
+
source: video,
|
|
714
|
+
face,
|
|
715
|
+
canvas: cropCanvas,
|
|
716
|
+
outputSize: { width: 224, height: 224 },
|
|
717
|
+
padding: 0.2,
|
|
718
|
+
});
|
|
719
|
+
```
|
|
720
|
+
|
|
721
|
+
Bounds use normalized tracker-input/source coordinates with a top-left origin.
|
|
722
|
+
`padding` is the fraction of detected width/height added on each side. The face
|
|
723
|
+
helper defaults to `0.2`; the generic helper defaults to `0`. The functions do
|
|
724
|
+
not mirror, classify, upload, or retain pixels. A `false` result means the
|
|
725
|
+
source dimensions, bounds, output dimensions, or canvas context were unusable.
|
|
726
|
+
Reuse one canvas on frame-driven paths. Face attribute expression should feed
|
|
727
|
+
this crop into its independent image classifier; never derive emotion labels
|
|
728
|
+
from `FaceInfo.blendshapes`.
|
|
729
|
+
|
|
730
|
+
### Face Actions — semantic face effect logic
|
|
731
|
+
|
|
732
|
+
Import `FACE_ACTION_NAMES`, `DEFAULT_FACE_ACTION_OPTIONS`,
|
|
733
|
+
`getFaceActionValue`, `hasFaceAction`, and `useFaceAction` from
|
|
734
|
+
`@vincentt-xr/sdk/tracking`. They derive from caller-owned `FaceInfo`; they do
|
|
735
|
+
not start a second tracker.
|
|
736
|
+
|
|
737
|
+
```tsx
|
|
738
|
+
const face = useFaceInfo();
|
|
739
|
+
const blink = useFaceAction(face, "eyeBlink");
|
|
740
|
+
|
|
741
|
+
useEffect(() => {
|
|
742
|
+
if (blink.justStarted) capturePhoto();
|
|
743
|
+
}, [blink.justStarted]);
|
|
744
|
+
|
|
745
|
+
const mouthAmount = face ? getFaceActionValue(face, "mouthOpen") : 0;
|
|
746
|
+
```
|
|
747
|
+
|
|
748
|
+
| Action | Source and direction | Default start / release / hold |
|
|
749
|
+
| ------------------------------------------------------------- | --------------------------------- | ------------------------------ |
|
|
750
|
+
| `browRaise` | max brow-up blendshape | `0.45 / 0.30 / 50 ms` |
|
|
751
|
+
| `eyeBlink`, `eyeBlinkLeft`, `eyeBlinkRight`, `eyeBlinkEither` | typed eye-blink blendshapes | `0.55 / 0.40 / 50 ms` |
|
|
752
|
+
| `mouthOpen` | `jawOpen` | `0.45 / 0.30 / 50 ms` |
|
|
753
|
+
| `mouthPout` | `mouthPucker` | `0.50 / 0.35 / 50 ms` |
|
|
754
|
+
| `headUp`, `headDown` | positive/negative pitch magnitude | `10° / 6° / 80 ms` |
|
|
755
|
+
| `headTurnLeft`, `headTurnRight` | positive/negative yaw magnitude | `10° / 6° / 80 ms` |
|
|
756
|
+
| `headTiltLeft`, `headTiltRight` | positive/negative roll magnitude | `10° / 6° / 80 ms` |
|
|
757
|
+
|
|
758
|
+
Blendshape values are `[0, 1]`; head values and thresholds are radians. Left
|
|
759
|
+
and up use the SDK-positive pose convention. Directional actions always return
|
|
760
|
+
a non-negative magnitude. Pass `neutralRotation: { ...face.pose.rotation }`
|
|
761
|
+
captured from an explicit user calibration gesture when the resting pose should
|
|
762
|
+
be treated as zero. Without it, head actions use zero rotation.
|
|
763
|
+
|
|
764
|
+
Use `getFaceActionValue` for continuous animation, `hasFaceAction` for a simple
|
|
765
|
+
level check, and `useFaceAction` for `active`, `justStarted`, `justEnded`, and
|
|
766
|
+
`phase: "begin" | "stay" | "end" | "none"`
|
|
767
|
+
with SDK hysteresis and hold timing. Do not guess blendshape strings, flip pose
|
|
768
|
+
signs, or rebuild local debounce/hysteresis logic. For multiple faces, select
|
|
769
|
+
one with `useFaceInfo({ faceIndex })`, or call
|
|
770
|
+
`getStrongestFaceAction(faces, action)` for an "any face" effect, then pass its
|
|
771
|
+
`face` into `useFaceAction`. The strongest selector is one pass and starts no
|
|
772
|
+
additional reader.
|
|
773
|
+
|
|
774
|
+
Effect House-style physical movement maps as follows: left/right/both/either
|
|
775
|
+
blink → `eyeBlinkLeft` / `eyeBlinkRight` / `eyeBlink` / `eyeBlinkEither`;
|
|
776
|
+
mouth open → `mouthOpen`; pout → `mouthPout`; eyebrow movement → `browRaise`.
|
|
777
|
+
Semantic blink left/right is viewer-facing and therefore swaps MediaPipe's raw
|
|
778
|
+
anatomical-side blendshape names. Never swap `FaceInfo.blendshapes` itself.
|
|
779
|
+
Every detected movement emits at least one `stay` frame after `begin` before
|
|
780
|
+
`end`.
|
|
781
|
+
These are observable movements, not image classification. Do not infer
|
|
782
|
+
`happy`, `sad`, `angry`, `fear`, `disgust`, `surprise`, or `neutral` from local
|
|
783
|
+
threshold combinations. Use the separate Face Attribute Expression API below.
|
|
784
|
+
|
|
785
|
+
Run `/face-actions` to inspect every live value, set neutral pose, tune the
|
|
786
|
+
selected action, and repeat the all-actions checklist. All actions and the
|
|
787
|
+
default thresholds were verified with the real-camera page on 26 July 2026.
|
|
788
|
+
Run the verified `/facial-movement-expression` page to test the Effect
|
|
789
|
+
House-compatible physical movement subset, phase edges, fixed-face or
|
|
790
|
+
strongest-face selection, and the image classifier. Its complete real-camera
|
|
791
|
+
flow passed on 26 July 2026.
|
|
792
|
+
|
|
793
|
+
### Face Attribute Expression — image-based classification
|
|
794
|
+
|
|
795
|
+
Import `useFaceAttributeExpression`, `FACE_ATTRIBUTE_EXPRESSION_NAMES`, and
|
|
796
|
+
their types from `@vincentt-xr/sdk/tracking`. Pass caller-owned `FaceInfo`; the
|
|
797
|
+
hook reuses the shared session input frame, crops that face, and lazy-loads the
|
|
798
|
+
OpenCV MobileFaceNet INT8 ONNX classifier. It does not create another face
|
|
799
|
+
reader or detector.
|
|
800
|
+
|
|
801
|
+
```tsx
|
|
802
|
+
const face = useFaceInfo();
|
|
803
|
+
const attribute = useFaceAttributeExpression(face, {
|
|
804
|
+
targetFps: 5,
|
|
805
|
+
minConfidence: 0.55,
|
|
806
|
+
});
|
|
807
|
+
|
|
808
|
+
if (attribute.status === "ready") {
|
|
809
|
+
const { expression, confidence, scores } = attribute.result;
|
|
810
|
+
}
|
|
811
|
+
```
|
|
812
|
+
|
|
813
|
+
The lifecycle is `idle | loading | ready | error`. Ready results contain
|
|
814
|
+
`expression`, `confidence`, and `scores`. `expression` can be `angry`,
|
|
815
|
+
`disgust`, `fear`, `happy`, `neutral`, `sad`,
|
|
816
|
+
`surprise`, or `unknown`. `scores` contains only the seven real model classes;
|
|
817
|
+
there is no fabricated `unknown` score. `unknown` is the low-confidence fallback
|
|
818
|
+
below `minConfidence`. Default `targetFps` is `5`; the model is about 1.36 MB,
|
|
819
|
+
loads outside the main bundle, and is SHA-384 verified. ONNX Runtime's WASM
|
|
820
|
+
worker files load from the jsDelivr package path that matches the active
|
|
821
|
+
`onnxruntime-web` runtime version, not through the consuming app's bundler. The
|
|
822
|
+
runtime files are fetched from jsDelivr without SDK-managed SRI because ONNX
|
|
823
|
+
Runtime resolves its own WASM assets.
|
|
824
|
+
|
|
825
|
+
For best threaded SIMD performance, production pages that run expression
|
|
826
|
+
classification should be cross-origin isolated
|
|
827
|
+
(`Cross-Origin-Opener-Policy: same-origin` and
|
|
828
|
+
`Cross-Origin-Embedder-Policy: require-corp`) so `SharedArrayBuffer` is
|
|
829
|
+
available. Without those headers, ONNX Runtime falls back to single-threaded
|
|
830
|
+
WASM.
|
|
831
|
+
|
|
832
|
+
`modelUrl` overrides must accept float32 NCHW `[1, 3, 112, 112]`, with planar
|
|
833
|
+
RGB normalized to `[-1, 1]`, and return seven logits ordered `angry`, `disgust`,
|
|
834
|
+
`fear`, `happy`, `neutral`, `sad`, `surprise`. Do not assume an arbitrary
|
|
835
|
+
MobileFaceNet ONNX model is compatible. Failed loads are removed from the
|
|
836
|
+
session cache so a remounted consumer can retry.
|
|
837
|
+
|
|
838
|
+
This is a visual-expression estimate, not ground-truth emotion, intent,
|
|
839
|
+
identity, or mental state. Never replace it with blendshape heuristics. Run the
|
|
840
|
+
verified `/facial-movement-expression` page to inspect every live model score.
|
|
841
|
+
|
|
842
|
+
### `<Axes>` — generic 3D orientation gizmo
|
|
843
|
+
|
|
844
|
+
Use `Axes` to visualize any 3D pose. X is red, Y is green, and Z is blue. Keep
|
|
845
|
+
tracking separate: bind a face, hand, or object pose through standard Three.js
|
|
846
|
+
`position` and `rotation` props.
|
|
847
|
+
|
|
848
|
+
```tsx
|
|
849
|
+
import { Axes } from "@vincentt-xr/sdk";
|
|
850
|
+
|
|
851
|
+
<Axes size={0.38} position={pose.position} rotation={pose.rotation} />;
|
|
852
|
+
```
|
|
853
|
+
|
|
135
854
|
### `<FaceTracker.Mesh />` / `<FaceMesh>` — Deforming 468-point face mesh
|
|
136
855
|
|
|
137
856
|
For masks, makeup, and full-face decoration that deforms with skin.
|
|
@@ -154,6 +873,10 @@ import { HandTracker } from "@vincentt-xr/sdk/tracking";
|
|
|
154
873
|
<HandTracker />;
|
|
155
874
|
```
|
|
156
875
|
|
|
876
|
+
Supports multiple detected hands: `hand` selects `"left"` or `"right"` when that
|
|
877
|
+
hand is available; `handIndex` falls back to positional selection. Existing
|
|
878
|
+
one-hand scenes keep working via the default first-hand selection.
|
|
879
|
+
|
|
157
880
|
### `<GestureTracker>` — Named gestures
|
|
158
881
|
|
|
159
882
|
The gesture door — the component that recognizes named gestures. Its `node.gesture` is one of these exact strings (copy verbatim — these are the only valid values):
|
|
@@ -232,6 +955,12 @@ Every tracker takes an `active` prop (default `true`). Set `active={false}` and
|
|
|
232
955
|
|
|
233
956
|
When several consumers share one model, it runs while **any** of them is active. `active` gates whether the model runs; `perf.targetFps` (below) caps how often it runs while active — use both. `<Segmentation>` also takes `active` (distinct from its `visible` prop, which only gates mask exposure while inference keeps running).
|
|
234
957
|
|
|
958
|
+
Session lifecycle preserves that ownership state: `session.stop()` pauses every
|
|
959
|
+
tracker without releasing its MediaPipe resources, and the next `session.start()`
|
|
960
|
+
resumes only models that still have at least one active owner. Static-image
|
|
961
|
+
detection pulses use the same suspension path, so source switches do not
|
|
962
|
+
accidentally reactivate an `active={false}` tracker.
|
|
963
|
+
|
|
235
964
|
### `perf` — cap inference rate + camera resolution
|
|
236
965
|
|
|
237
966
|
Set `perf` on `<XRProvider>` to trade tracking fidelity for lower load and heat. This is the single biggest lever if the device runs hot.
|
|
@@ -257,6 +986,11 @@ Per-tracker override beats the session default:
|
|
|
257
986
|
|
|
258
987
|
The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — good for both dev laptops and end-user phones. Don't lower it further without a reason; don't remove it to "get full quality" unless you know the target device can take the heat.
|
|
259
988
|
|
|
989
|
+
SDK-owned tracker, occluder, and canonical-face assets are SHA-384 verified
|
|
990
|
+
against hashes pinned in the package. A missing hash for
|
|
991
|
+
`https://cdn.vincentt.studio` fails closed; custom consumer-owned asset URLs
|
|
992
|
+
remain best-effort and warn when no pinned hash exists.
|
|
993
|
+
|
|
260
994
|
---
|
|
261
995
|
|
|
262
996
|
## Reading tracker state
|
|
@@ -265,9 +999,11 @@ The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — g
|
|
|
265
999
|
|
|
266
1000
|
Wraps children; the child's world position is updated to the tracked landmark each frame. Use for all decoration that simply follows a face/hand/body point.
|
|
267
1001
|
|
|
268
|
-
Valid targets
|
|
1002
|
+
Valid targets are centralized in `FACE_TARGETS`; use that exported list for
|
|
1003
|
+
editor dropdowns instead of copying it. Left/right mean the tracked person's
|
|
1004
|
+
perspective.
|
|
269
1005
|
|
|
270
|
-
- **Face:** `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`
|
|
1006
|
+
- **Face binding presets:** `face.center`, `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEye`, `face.rightEye`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`, `face.leftEarlobe`, `face.rightEarlobe`, `face.custom`
|
|
271
1007
|
- **Hand:** `hand.wrist`, `hand.thumbTip`, `hand.indexTip`, `hand.middleTip`, `hand.ringTip`, `hand.pinkyTip`, `hand.indexBase`, `hand.middleBase`, `hand.ringBase`, `hand.pinkyBase`
|
|
272
1008
|
- **Body:** `body.head`, `body.neck`, `body.leftShoulder`, `body.rightShoulder`, `body.leftElbow`, `body.rightElbow`, `body.leftWrist`, `body.rightWrist`, `body.leftHip`, `body.rightHip`, `body.leftKnee`, `body.rightKnee`, `body.leftAnkle`, `body.rightAnkle`
|
|
273
1009
|
|
|
@@ -282,6 +1018,32 @@ Valid targets:
|
|
|
282
1018
|
</FaceTracker>
|
|
283
1019
|
```
|
|
284
1020
|
|
|
1021
|
+
`face.center` is the general default. Eye targets are composite eye centers;
|
|
1022
|
+
`*EyeOuter` targets are exact outer-corner landmarks. Earlobe targets are the
|
|
1023
|
+
nearest stable MediaPipe face-oval points, not anatomical earlobe detections.
|
|
1024
|
+
All Effect House `FaceBindingAnchorType` names therefore have a Vincentt
|
|
1025
|
+
equivalent, but their numeric canonical-mesh coordinates are not compatible.
|
|
1026
|
+
|
|
1027
|
+
For `face.custom`, pass normalized face-local coordinates. The origin is face
|
|
1028
|
+
center; X runs from person's left `-1` to right `+1`, Y from chin `-1` to
|
|
1029
|
+
forehead `+1`, and positive Z comes toward the camera.
|
|
1030
|
+
|
|
1031
|
+
```tsx
|
|
1032
|
+
<FaceTracker>
|
|
1033
|
+
<TrackingAnchor target="face.custom" customAnchorPoint={{ x: 0.7, y: 0.2, z: 0 }} smoothing={15}>
|
|
1034
|
+
<mesh />
|
|
1035
|
+
</TrackingAnchor>
|
|
1036
|
+
</FaceTracker>
|
|
1037
|
+
```
|
|
1038
|
+
|
|
1039
|
+
Scene objects use the same `TrackingAnchor` component settings. A 2D null with
|
|
1040
|
+
`ScreenTransform` + `TrackingAnchor` projects the preset into SDK design pixels,
|
|
1041
|
+
so all child Screen Shapes move as one group. Use `/face-binding-anchors` to
|
|
1042
|
+
test every preset on the editable `rose null`; the inspector target options
|
|
1043
|
+
come directly from `FACE_TARGETS`. Toggle `Follow Face Roll` to rotate the
|
|
1044
|
+
attachment with head tilt. The example hierarchy intentionally shows only the
|
|
1045
|
+
combined root object.
|
|
1046
|
+
|
|
285
1047
|
**`smoothing` — smooth motion from throttled inference.** Inference is throttled (the `perf` API caps it at 24fps by default) while rendering runs at 60-120fps, so a raw anchor holds then jumps on each new sample — visible stepping under head/hand movement. A plain lerp only hides it at rest; under motion it still steps. Pass `smoothing` (`~15` is a good default) and the anchor runs a **One Euro filter** — the standard adaptive low-pass for interactive tracking: it smooths hard when the landmark moves slowly (kills jitter) and loosens as it speeds up (kills lag), and never extrapolates so it can't overshoot or wobble. Higher `smoothing` is snappier (less lag, slightly more jitter); lower is smoother. It snaps (resets the filter) on first placement and on re-acquire, so it never slides in from a stale position. `0` (default) keeps the raw snap. Rotation is not smoothed. Prefer this over a hand-rolled `useFrame` lerp — a hand-rolled lerp reintroduces the stepping.
|
|
286
1048
|
|
|
287
1049
|
```tsx
|
|
@@ -350,10 +1112,101 @@ import { GestureTracker, GestureTrigger } from "@vincentt-xr/sdk/tracking";
|
|
|
350
1112
|
|
|
351
1113
|
In a v2-template project this is already mounted in `src/App.tsx` — do NOT add a second one there. Mentioned here because `customBackground` is how you do background replacement: pair it with `<Segmentation type="portrait" />` and pass the segmentation mask texture as `segmentationMask`.
|
|
352
1114
|
|
|
1115
|
+
`VideoBackground` is a flat camera-background component: its material keeps
|
|
1116
|
+
`depthTest={false}` and `depthWrite={false}`. Editor-style 3D viewports that
|
|
1117
|
+
treat the camera feed as a real scene plane should render a separate editor
|
|
1118
|
+
video plane instead of changing `VideoBackground` semantics.
|
|
1119
|
+
|
|
353
1120
|
---
|
|
354
1121
|
|
|
355
1122
|
## Mesh primitives and textures
|
|
356
1123
|
|
|
1124
|
+
`ScreenImage` keeps fitted clones synchronized for dynamic video and canvas textures.
|
|
1125
|
+
|
|
1126
|
+
### Texture Types API usage
|
|
1127
|
+
|
|
1128
|
+
`ScreenImage` is the common render target for static and animated texture sources. Use
|
|
1129
|
+
the normal runtime import for rendering:
|
|
1130
|
+
|
|
1131
|
+
```tsx
|
|
1132
|
+
import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
|
|
1133
|
+
|
|
1134
|
+
<ScreenSpaceUI>
|
|
1135
|
+
<ScreenImage
|
|
1136
|
+
source={{ kind: "image", src: imageUrl }}
|
|
1137
|
+
materialType="unlit"
|
|
1138
|
+
stretchMode="fit"
|
|
1139
|
+
transparent
|
|
1140
|
+
/>
|
|
1141
|
+
</ScreenSpaceUI>;
|
|
1142
|
+
```
|
|
1143
|
+
|
|
1144
|
+
Supported source behavior:
|
|
1145
|
+
|
|
1146
|
+
- Image: static URL texture. No playback controls.
|
|
1147
|
+
- Runtime texture: pass `{ kind: "texture", texture }` when supplying an existing `THREE.Texture`.
|
|
1148
|
+
- Video: use a managed `THREE.VideoTexture`; support play/pause, restart, frame stepping, playback rate, forward, and reverse stepping. Negative `playbackRate` is not used.
|
|
1149
|
+
- GIF: decode frames into a canvas texture. Preserve GIF dimensions and per-frame delays before applying ScreenImage fitting.
|
|
1150
|
+
- Image sequence: draw loaded frames into a canvas texture using FPS and frame state.
|
|
1151
|
+
|
|
1152
|
+
Animated sources support forward, reverse, ping-pong, random, looping, finite loop
|
|
1153
|
+
counts, pause/resume at current frame, restart, frame status, and ended state where
|
|
1154
|
+
the source supports it. Static image sources must not expose animation controls.
|
|
1155
|
+
|
|
1156
|
+
ScreenImage inspector rules:
|
|
1157
|
+
|
|
1158
|
+
- `ScreenImageSettingsPanel` must read/write the same scene-object state used by the rendered `ScreenImage`.
|
|
1159
|
+
- Transform drag/place must be wired through `onScreenTransformChange`.
|
|
1160
|
+
- Use `mapScreenImageSettingsToProps` to convert inspector canvas coordinates into render props.
|
|
1161
|
+
- Do not pass inspector pixel dimensions directly as world-space `ScreenImage` dimensions.
|
|
1162
|
+
- Dynamic canvas/video texture updates must mark fitted texture clones for GPU upload.
|
|
1163
|
+
|
|
1164
|
+
Low-level texture integration imports from `@vincentt-xr/sdk/low-level`:
|
|
1165
|
+
|
|
1166
|
+
```ts
|
|
1167
|
+
import {
|
|
1168
|
+
BasicAnimatedTextureController,
|
|
1169
|
+
createTextureSourceResolver,
|
|
1170
|
+
getPlaybackFrame,
|
|
1171
|
+
getSequenceFrame,
|
|
1172
|
+
type AnimatedTextureController,
|
|
1173
|
+
type AnimatedTexturePlayMode,
|
|
1174
|
+
type ResolvedTextureSource,
|
|
1175
|
+
type TextureSourceResolver,
|
|
1176
|
+
type TextureSourceProvider,
|
|
1177
|
+
} from "@vincentt-xr/sdk/low-level";
|
|
1178
|
+
```
|
|
1179
|
+
|
|
1180
|
+
Controller lifecycle is: `play()`, `pause()`, `restart()`, optional `seek()`,
|
|
1181
|
+
`setPlayMode("forward" | "reverse" | "ping-pong" | "random")`,
|
|
1182
|
+
`setLoop(enabled, count)`, `subscribe(listener)`, then `dispose()`. Read
|
|
1183
|
+
`getSnapshot()` for state, frame/time, duration, frame count, loop, and errors;
|
|
1184
|
+
check `capabilities` before rendering a control.
|
|
1185
|
+
|
|
1186
|
+
Frame helpers are also available from the same `/low-level` door:
|
|
1187
|
+
`getPlaybackFrame(tick, frameCount, mode, loop, loopCount)` handles forward,
|
|
1188
|
+
reverse, ping-pong, random, loop, and ended math; `getSequenceFrame(elapsedSeconds,
|
|
1189
|
+
fps, frameCount)` maps elapsed time to a bounded sequence index.
|
|
1190
|
+
|
|
1191
|
+
Event compatibility: subscribe with `controller.on(type, listener)` and remove
|
|
1192
|
+
with the returned disposer or `off()`. Supported event types are
|
|
1193
|
+
`KeyFrameInSecond`, `Pause`, `PlayBegin`, `PlayEnd`, `Resume`, and `Stop`.
|
|
1194
|
+
Event snapshots include the current frame/time, playback state, loop settings,
|
|
1195
|
+
and optional duration/frame count. Use capability checks before exposing an
|
|
1196
|
+
event-driven control.
|
|
1197
|
+
|
|
1198
|
+
Consumer example: `docs/texture-api-consumer.md`.
|
|
1199
|
+
|
|
1200
|
+
Browser verification checklist: Chrome/Android (autoplay requires muted media),
|
|
1201
|
+
Safari/iOS (inline video requires `playsInline` and user-gesture fallback), and
|
|
1202
|
+
background tabs (timers may throttle; frame state must remain bounded). Validate
|
|
1203
|
+
source switching, pause-at-current-frame, loop-off ending, reverse/ping-pong,
|
|
1204
|
+
stretch modes, drag/place transforms, and unmount cleanup in each target.
|
|
1205
|
+
|
|
1206
|
+
`ResolvedTextureSource` owns a texture, optional controller, metadata, and
|
|
1207
|
+
ownership-safe `dispose()`. Abort stale loads, stop timers/RAF, remove media
|
|
1208
|
+
listeners, and never dispose externally owned textures.
|
|
1209
|
+
|
|
357
1210
|
Use R3F intrinsic `<mesh>`, `<group>`, `<planeGeometry>`, `<meshBasicMaterial>`, etc. Load image textures with drei's `useTexture`.
|
|
358
1211
|
|
|
359
1212
|
**Always name your meshes** (`name="myLabel"`). Names enable runtime lookup AND visual-feedback editing by the editor.
|
|
@@ -373,29 +1226,790 @@ const tex = useTexture("https://cdn.../badge.webp");
|
|
|
373
1226
|
|
|
374
1227
|
---
|
|
375
1228
|
|
|
1229
|
+
## Scene-object runtime and authoring state
|
|
1230
|
+
|
|
1231
|
+
Runtime components import from `@vincentt-xr/sdk`. Scene-object data and store
|
|
1232
|
+
operations import from `@vincentt-xr/sdk/scene-object`. DOM editors import from
|
|
1233
|
+
`@vincentt-xr/sdk/debug-ui`.
|
|
1234
|
+
|
|
1235
|
+
### `<Transform3D value={settings}>` and `<MeshRenderer value={settings}>`
|
|
1236
|
+
|
|
1237
|
+
`Transform3D` applies position, rotation (degrees), scale, visibility, and optional
|
|
1238
|
+
Drei transform controls to its children. `MeshRenderer` renders a cube, sphere,
|
|
1239
|
+
plane, or first-pass GLTF/GLB asset with basic/standard material settings.
|
|
1240
|
+
|
|
1241
|
+
```tsx
|
|
1242
|
+
import {
|
|
1243
|
+
DEFAULT_MESH_RENDERER,
|
|
1244
|
+
DEFAULT_TRANSFORM_3D,
|
|
1245
|
+
MeshRenderer,
|
|
1246
|
+
Transform3D,
|
|
1247
|
+
} from "@vincentt-xr/sdk";
|
|
1248
|
+
|
|
1249
|
+
<Transform3D
|
|
1250
|
+
value={{
|
|
1251
|
+
...DEFAULT_TRANSFORM_3D,
|
|
1252
|
+
position: { x: 0, y: 0, z: -1 },
|
|
1253
|
+
showTransformControls: true,
|
|
1254
|
+
}}
|
|
1255
|
+
onChange={setTransform}
|
|
1256
|
+
>
|
|
1257
|
+
<MeshRenderer
|
|
1258
|
+
value={{
|
|
1259
|
+
...DEFAULT_MESH_RENDERER,
|
|
1260
|
+
material: { ...DEFAULT_MESH_RENDERER.material, color: "#38bdf8" },
|
|
1261
|
+
}}
|
|
1262
|
+
/>
|
|
1263
|
+
</Transform3D>;
|
|
1264
|
+
```
|
|
1265
|
+
|
|
1266
|
+
`Transform3DProps` is `{ value, onChange?, children }`.
|
|
1267
|
+
`MeshRendererProps` is `{ value, renderOrder? }`. `renderOrder` is applied to the
|
|
1268
|
+
renderable mesh itself. Disabled or invisible settings render nothing.
|
|
1269
|
+
|
|
1270
|
+
`MeshRendererSettings` fields:
|
|
1271
|
+
|
|
1272
|
+
- `enabled`, `visible`: gate rendering.
|
|
1273
|
+
- `mesh`: `"cube" | "sphere" | "plane"` for primitive fallback.
|
|
1274
|
+
- `assetUrl?`: optional GLTF/GLB URL. When present, the asset renders instead of
|
|
1275
|
+
the primitive mesh.
|
|
1276
|
+
- `assetNodeName?`: optional object name inside the loaded GLTF scene. When
|
|
1277
|
+
missing or unresolved, the whole cloned scene renders.
|
|
1278
|
+
- `material.type`: `"standard" | "basic"`.
|
|
1279
|
+
- `material.color`, `opacity`, `wireframe`, `roughness`, `metalness`: material
|
|
1280
|
+
overrides. Wireframe belongs to material state, not geometry state.
|
|
1281
|
+
- `material.renderState?`: optional advanced draw behavior:
|
|
1282
|
+
- `depthTest`, `depthWrite`, `colorWrite`: booleans. `colorWrite` is the
|
|
1283
|
+
all-channel write gate.
|
|
1284
|
+
- `colorMask`: `{ r, g, b, a }` per-channel color write gates. This is
|
|
1285
|
+
applied as scoped WebGL color-mask state for each mesh draw and restored
|
|
1286
|
+
after the object renders.
|
|
1287
|
+
- `depthFunc`: `"never" | "less" | "equal" | "lessEqual" | "greater" |
|
|
1288
|
+
"notEqual" | "greaterEqual" | "always"`.
|
|
1289
|
+
- `cullMode`: `"back" | "front" | "off"`. The default `"back"` maps to
|
|
1290
|
+
Three.js `FrontSide`; `"off"` maps to `DoubleSide`.
|
|
1291
|
+
- `blendMode`: `"normal" | "additive" | "subtractive" | "multiply" |
|
|
1292
|
+
"none"`.
|
|
1293
|
+
- `polygonOffset`: `{ enabled, factor, units }`.
|
|
1294
|
+
- `shadowMode`: `"off" | "cast" | "receive" | "castReceive"`.
|
|
1295
|
+
|
|
1296
|
+
Current GLTF support is intentionally first-pass: the renderer clones the loaded
|
|
1297
|
+
scene and applies color, opacity, wireframe, render-state fields, and shadow
|
|
1298
|
+
flags to cloned mesh materials.
|
|
1299
|
+
Do not rely on it yet for advanced material graph preservation, animation,
|
|
1300
|
+
skinning behavior, or production asset lifecycle guarantees.
|
|
1301
|
+
|
|
1302
|
+
Render-state defaults preserve current behavior: depth test/write on,
|
|
1303
|
+
`lessEqual` depth function, color write on, back-face culling, normal blending,
|
|
1304
|
+
RGBA color mask fully enabled, and polygon offset disabled. Instance count is not
|
|
1305
|
+
part of `MeshRenderer.material.renderState`; it belongs to instanced mesh
|
|
1306
|
+
rendering rather than material shading state.
|
|
1307
|
+
|
|
1308
|
+
`assetUrl` uses Drei `useGLTF`, so it suspends while loading. `MeshRenderer`
|
|
1309
|
+
wraps asset-backed rendering in `Suspense fallback={null}` so data-driven
|
|
1310
|
+
`SceneObjectRenderer` usage is safe by default. Add your own outer `Suspense`
|
|
1311
|
+
or call `useGLTF.preload(url)` only when you want a visible loading state or
|
|
1312
|
+
early fetch. Primitive meshes do not require Suspense.
|
|
1313
|
+
|
|
1314
|
+
### Screen Transform data and resolver
|
|
1315
|
+
|
|
1316
|
+
Import `ScreenTransform2DSettings`, `DEFAULT_SCREEN_TRANSFORM_2D`,
|
|
1317
|
+
`DEFAULT_SCREEN_TRANSFORM_ROOT`, `DEFAULT_REFERENCE_PIXELS_PER_UNIT`,
|
|
1318
|
+
`mergeScreenTransform2D`, and `resolveScreenTransform2D` from the core door.
|
|
1319
|
+
|
|
1320
|
+
`ScreenTransform2DSettings` owns enabled/visible state, canvas-pixel position and
|
|
1321
|
+
size, normalized pivot, degree rotation, 2D scale, reference pixels per unit,
|
|
1322
|
+
render order, overlay mode, and guide visibility.
|
|
1323
|
+
|
|
1324
|
+
```ts
|
|
1325
|
+
import {
|
|
1326
|
+
DEFAULT_SCREEN_TRANSFORM_2D,
|
|
1327
|
+
mergeScreenTransform2D,
|
|
1328
|
+
resolveScreenTransform2D,
|
|
1329
|
+
type ScreenTransform2DSettings,
|
|
1330
|
+
} from "@vincentt-xr/sdk";
|
|
1331
|
+
|
|
1332
|
+
const transform: ScreenTransform2DSettings = mergeScreenTransform2D({
|
|
1333
|
+
position: { x: 0, y: 240 },
|
|
1334
|
+
size: { width: 420, height: 420 },
|
|
1335
|
+
showTransformGuides: true,
|
|
1336
|
+
});
|
|
1337
|
+
|
|
1338
|
+
const resolved = resolveScreenTransform2D({
|
|
1339
|
+
transform,
|
|
1340
|
+
parentSize: { width: 9, height: 16 },
|
|
1341
|
+
});
|
|
1342
|
+
```
|
|
1343
|
+
|
|
1344
|
+
Use `mergeScreenTransform2D` to fill nested defaults. Use
|
|
1345
|
+
`resolveScreenTransform2D` whenever runtime or editor tooling must convert the
|
|
1346
|
+
authoring model into center, size, rotation, pivot offset, and aspect. Do not copy
|
|
1347
|
+
the default `720x1280` root or `32` reference-pixel constant into another renderer
|
|
1348
|
+
or guide.
|
|
1349
|
+
|
|
1350
|
+
### Screen/world coordinate utilities
|
|
1351
|
+
|
|
1352
|
+
Import the `*ScreenPixels` coordinate helpers, `screenPointToRay`, and
|
|
1353
|
+
`smoothNumber` / `smoothVector2` / `smoothVector3` from `@vincentt-xr/sdk`. Use
|
|
1354
|
+
these shared helpers instead of rewriting mapping or projection math or assuming
|
|
1355
|
+
design pixels are Three.js world units.
|
|
1356
|
+
SDK screen pixels use the Effect House-style `720x1280` design canvas, center
|
|
1357
|
+
origin, and +Y up.
|
|
1358
|
+
|
|
1359
|
+
```ts
|
|
1360
|
+
const units = screenPixelsToUnits(120, { parentSize: screenDimensions });
|
|
1361
|
+
const pixels = screenUnitsToPixels(units, { parentSize: screenDimensions });
|
|
1362
|
+
|
|
1363
|
+
const labelPosition = worldToScreenPixels({
|
|
1364
|
+
point: trackedObject.getWorldPosition(new THREE.Vector3()),
|
|
1365
|
+
camera,
|
|
1366
|
+
viewportSize,
|
|
1367
|
+
});
|
|
1368
|
+
|
|
1369
|
+
const placementRay = screenPointToRay({
|
|
1370
|
+
point: { x: tapX, y: tapY },
|
|
1371
|
+
camera,
|
|
1372
|
+
viewportSize,
|
|
1373
|
+
});
|
|
1374
|
+
```
|
|
1375
|
+
|
|
1376
|
+
- Use pixel/unit conversion for Screen Image, Screen Text, Screen Shape, editor
|
|
1377
|
+
handles, and custom `ScreenSpaceUI` content.
|
|
1378
|
+
- Use `worldToScreenPixels` for labels, prompts, and indicators that follow a
|
|
1379
|
+
face, hand, body joint, or 3D object. Its `depth` is projected NDC depth; hide
|
|
1380
|
+
the result when `depth` is outside `[-1, 1]` because it is outside the camera
|
|
1381
|
+
clip range.
|
|
1382
|
+
- Use `screenPointToRay` for tap selection, hit testing, and 3D placement. Pass
|
|
1383
|
+
design-canvas coordinates, not DOM client pixels.
|
|
1384
|
+
- Use `clientToScreenPixels` / `screenPixelsToClient` for pointer events and DOM
|
|
1385
|
+
overlays; pass the viewport element's `getBoundingClientRect()` result.
|
|
1386
|
+
- Use `normalizedToScreenPixels` / `screenPixelsToNormalized` for normalized
|
|
1387
|
+
viewport coordinates with top-left origin. These values are not automatically
|
|
1388
|
+
camera-crop adjusted.
|
|
1389
|
+
- Use `mediaToScreenPixels` / `screenPixelsToMedia` with the shared
|
|
1390
|
+
`XRMediaLayout` for camera/image pixels. They apply cover/contain offsets,
|
|
1391
|
+
scaling, and optional display mirroring; do not manually repeat that math.
|
|
1392
|
+
- Use `smoothNumber`, `smoothVector2`, or `smoothVector3` for tracked animation.
|
|
1393
|
+
Pass the R3F frame `delta` in seconds and a smoothing rate (larger is faster).
|
|
1394
|
+
Reuse the optional `out` vector in per-frame paths to avoid allocations.
|
|
1395
|
+
- Zero-size viewport/parent inputs return neutral zero coordinates or units so
|
|
1396
|
+
first paint and collapsed layouts do not throw. Invalid custom root sizes and
|
|
1397
|
+
`referencePixelsPerUnit` still throw because they are configuration errors.
|
|
1398
|
+
- Use Three.js `localToWorld`, `worldToLocal`, `Vector3.project/unproject`,
|
|
1399
|
+
`set`, `toArray`, and `fromArray` directly; do not add SDK wrappers for them.
|
|
1400
|
+
|
|
1401
|
+
### Scene-object store
|
|
1402
|
+
|
|
1403
|
+
```ts
|
|
1404
|
+
import {
|
|
1405
|
+
addComponentToObject,
|
|
1406
|
+
createDefaultComponent,
|
|
1407
|
+
createSceneObject,
|
|
1408
|
+
deleteSceneObject,
|
|
1409
|
+
getOrderedComponents,
|
|
1410
|
+
moveComponent,
|
|
1411
|
+
removeComponent,
|
|
1412
|
+
type SceneObjectStore,
|
|
1413
|
+
} from "@vincentt-xr/sdk/scene-object";
|
|
1414
|
+
|
|
1415
|
+
const cube = createSceneObject({ name: "Cube" });
|
|
1416
|
+
const transform = createDefaultComponent("Transform", cube.id);
|
|
1417
|
+
const mesh = createDefaultComponent("MeshRenderer", cube.id);
|
|
1418
|
+
cube.componentIds = [transform.id, mesh.id];
|
|
1419
|
+
|
|
1420
|
+
const store: SceneObjectStore = {
|
|
1421
|
+
objects: { [cube.id]: cube },
|
|
1422
|
+
components: { [transform.id]: transform, [mesh.id]: mesh },
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1425
|
+
const ordered = getOrderedComponents(store, cube.id);
|
|
1426
|
+
```
|
|
1427
|
+
|
|
1428
|
+
`SceneObject` owns identity, display name, enabled state, optional `renderGroupId`,
|
|
1429
|
+
optional hierarchy fields, and ordered `componentIds`. `SceneComponent` owns one
|
|
1430
|
+
typed settings payload and points back to its object. `ComponentRegistry` maps
|
|
1431
|
+
component types to those settings.
|
|
1432
|
+
|
|
1433
|
+
Persisted scenes may still provide the deprecated `layerId`; migration treats it
|
|
1434
|
+
as a fallback when `renderGroupId` is absent. New code should only write
|
|
1435
|
+
`renderGroupId`.
|
|
1436
|
+
|
|
1437
|
+
Nested scene-object fields:
|
|
1438
|
+
|
|
1439
|
+
- `parentObjectId?: SceneObjectId | null` points to the parent object.
|
|
1440
|
+
- `childObjectIds?: SceneObjectId[]` stores ordered child IDs for hierarchy and
|
|
1441
|
+
runtime recursion.
|
|
1442
|
+
|
|
1443
|
+
Nested transforms are evaluated through the runtime object tree. Child objects
|
|
1444
|
+
render inside the parent object, so parent movement affects descendants while
|
|
1445
|
+
each child keeps its own local component settings.
|
|
1446
|
+
|
|
1447
|
+
Store helpers are immutable and preserve extra fields on an app-extended store:
|
|
1448
|
+
|
|
1449
|
+
- `addComponentToObject(store, component)` adds the component and its ID.
|
|
1450
|
+
- `moveComponent(store, objectId, componentId, nextIndex)` reorders one object.
|
|
1451
|
+
- `removeComponent(...)` removes the component and its ID.
|
|
1452
|
+
- `deleteSceneObject(store, objectId)` removes the object and all owned components.
|
|
1453
|
+
- `replaceSceneComponent(store, component)` replaces one component.
|
|
1454
|
+
- `replaceSceneObject(store, object)` replaces one object.
|
|
1455
|
+
- `setSceneObjectVisible(store, objectId, visible)` toggles object `enabled`.
|
|
1456
|
+
- `getSceneObjectChildren(store, objectId)` resolves explicit and implicit child
|
|
1457
|
+
objects in hierarchy order.
|
|
1458
|
+
- `setSceneObjectParent(store, objectId, parentObjectId, nextIndex?)` updates both
|
|
1459
|
+
sides of a parent/child relationship and rejects cycles or mixed transform
|
|
1460
|
+
domains. Parent `Transform` under `Transform`, and `ScreenTransform` under
|
|
1461
|
+
`ScreenTransform`. Existing child values become local-to-parent values.
|
|
1462
|
+
- `setSceneObjectParentPreserveWorldTransform(store, objectId, parentObjectId,
|
|
1463
|
+
nextIndex?)` is the editor seam for drag/drop and parent dropdowns. It updates
|
|
1464
|
+
the hierarchy and recalculates the child's local `Transform` or
|
|
1465
|
+
`ScreenTransform` so the object does not visually jump when nested or
|
|
1466
|
+
un-nested. This is exact for representable transforms. Rotation combined with
|
|
1467
|
+
non-uniform ancestor scale introduces shear, which neither component stores,
|
|
1468
|
+
so preservation in that case is approximate.
|
|
1469
|
+
- `createOrderedComponentsSelector()` creates a memoized ordered selector.
|
|
1470
|
+
|
|
1471
|
+
### `<SceneObjectRenderer>`
|
|
1472
|
+
|
|
1473
|
+
`SceneObjectRenderer` reads an object plus its ordered components and renders the
|
|
1474
|
+
supported composition. Import the renderer from core and its store type from
|
|
1475
|
+
`/scene-object`.
|
|
1476
|
+
|
|
1477
|
+
```tsx
|
|
1478
|
+
import { SceneObjectRenderer } from "@vincentt-xr/sdk";
|
|
1479
|
+
import type { SceneComponent, SceneObjectStore } from "@vincentt-xr/sdk/scene-object";
|
|
1480
|
+
|
|
1481
|
+
<SceneObjectRenderer
|
|
1482
|
+
objectId={cube.id}
|
|
1483
|
+
store={store}
|
|
1484
|
+
editorMode
|
|
1485
|
+
onComponentChange={(component: SceneComponent) => replaceComponent(component)}
|
|
1486
|
+
/>;
|
|
1487
|
+
```
|
|
1488
|
+
|
|
1489
|
+
Props are `objectId`, `store`, optional `editorMode`, optional
|
|
1490
|
+
`onComponentChange`, and optional `fallback`. Supported compositions are:
|
|
1491
|
+
|
|
1492
|
+
- `Transform` + `MeshRenderer`
|
|
1493
|
+
- `Transform` + `Light`
|
|
1494
|
+
- `Transform` + `MeshRenderer` with optional `LookAt` and `PinToMesh`
|
|
1495
|
+
- `Transform` only as an invisible 3D null/container
|
|
1496
|
+
- `ScreenTransform` only as an invisible 2D null/container
|
|
1497
|
+
- `ScreenTransform` + `ScreenImage`
|
|
1498
|
+
- `ScreenTransform` + `ScreenText`
|
|
1499
|
+
|
|
1500
|
+
For mesh objects, `SceneObjectRenderer` registers the object id at runtime so
|
|
1501
|
+
behavior components can resolve serialized target ids. Inspector/editor changes
|
|
1502
|
+
must route through `onComponentChange` and replace the selected object's
|
|
1503
|
+
component in the store; do not keep separate page-local copies of transform,
|
|
1504
|
+
material, light, LookAt, or PinToMesh state.
|
|
1505
|
+
|
|
1506
|
+
In editor mode, transform changes return through `onComponentChange`.
|
|
1507
|
+
Unsupported compositions render nothing in production and use `fallback` in
|
|
1508
|
+
editor mode.
|
|
1509
|
+
|
|
1510
|
+
---
|
|
1511
|
+
|
|
1512
|
+
## `<ScreenText>` — Standard screen-space text
|
|
1513
|
+
|
|
1514
|
+
Import from `@vincentt-xr/sdk`. Use this when you want the standard
|
|
1515
|
+
screen-text lane instead of wiring `ScreenTransform` + `TextLabel` yourself.
|
|
1516
|
+
For future Vincentt scenes, prefer `ScreenText` over raw `TextLabel` for end-user
|
|
1517
|
+
screen copy such as prompts, captions, banners, badges, and titles.
|
|
1518
|
+
|
|
1519
|
+
`ScreenText` composes:
|
|
1520
|
+
|
|
1521
|
+
- full-width screen-space lane presets
|
|
1522
|
+
- `verticalAlign: "top" | "center" | "bottom"`
|
|
1523
|
+
- `TextLabel` rendering props such as `textAlign`, overflow, background, stroke, and shadow
|
|
1524
|
+
|
|
1525
|
+
```tsx
|
|
1526
|
+
import { ScreenSpaceUI, ScreenText } from "@vincentt-xr/sdk";
|
|
1527
|
+
|
|
1528
|
+
<ScreenSpaceUI>
|
|
1529
|
+
<ScreenText
|
|
1530
|
+
name="headline"
|
|
1531
|
+
text="Head-Tilt Trivia"
|
|
1532
|
+
coordinateSpace="canvas"
|
|
1533
|
+
canvasSize={{ width: 720, height: 1280 }}
|
|
1534
|
+
position={[0, 500]}
|
|
1535
|
+
size={[620, 140]}
|
|
1536
|
+
textAlign="center"
|
|
1537
|
+
fontSize={270}
|
|
1538
|
+
bgFit="text"
|
|
1539
|
+
bgColor="transparent"
|
|
1540
|
+
/>
|
|
1541
|
+
</ScreenSpaceUI>;
|
|
1542
|
+
```
|
|
1543
|
+
|
|
1544
|
+
`ScreenText` supports the same screen coordinate contract as `ScreenImage`:
|
|
1545
|
+
`coordinateSpace="normalized"` uses normalized screen units, while
|
|
1546
|
+
`coordinateSpace="canvas"` uses Effect House-style design-canvas pixels
|
|
1547
|
+
(`720x1280` by default, center origin, +y up). Its `size` is the text bounding
|
|
1548
|
+
rectangle. The text renderer then aligns, shrinks, clips, or backgrounds the
|
|
1549
|
+
text inside that rectangle.
|
|
1550
|
+
|
|
1551
|
+
Use `ScreenText` when:
|
|
1552
|
+
|
|
1553
|
+
- you want top / center / bottom screen placement
|
|
1554
|
+
- you want wide-lane text alignment without hand-authoring anchors
|
|
1555
|
+
- you want the default end-user screen text behavior
|
|
1556
|
+
|
|
1557
|
+
Use `TextLabel` when:
|
|
1558
|
+
|
|
1559
|
+
- you need raw text rendering outside the screen-text presets
|
|
1560
|
+
- you want to place text manually in world space or inside a custom transform
|
|
1561
|
+
- you are building a lower-level composite where another component owns layout
|
|
1562
|
+
|
|
1563
|
+
`SceneObjectRenderer` owns the required `ScreenSpaceUI` container when it renders
|
|
1564
|
+
an object composed from `ScreenTransform` + `ScreenText` or `ScreenImage`. Do not
|
|
1565
|
+
wrap those renderer instances in an additional `ScreenSpaceUI`; direct uses of
|
|
1566
|
+
`ScreenText` and `ScreenImage` still require the container shown above.
|
|
1567
|
+
|
|
1568
|
+
Debug panels are independent, selection-driven surfaces. Use `HierarchyPanel` for a
|
|
1569
|
+
render-group-aware scene-object list and render one
|
|
1570
|
+
`ObjectInspectorPanel` for the selected object. Both accept `initialPosition` so the
|
|
1571
|
+
hierarchy and Inspector can be presented side by side instead of overlapping.
|
|
1572
|
+
`ScreenTextSettingsPanel` and `ScreenImageSettingsPanel` already own their complete
|
|
1573
|
+
Inspector UI; pass their optional `inspector` binding when they edit an object from
|
|
1574
|
+
a shared scene store. Do not nest either one inside another `ObjectInspectorPanel`.
|
|
1575
|
+
`ObjectInspectorPanel` is uncontrolled by default, but multi-object editors can pass
|
|
1576
|
+
`position` and `onPositionChange` to preserve one Inspector location while swapping
|
|
1577
|
+
the selected object's panel. `SceneObjectRenderer` forwards screen transform-guide
|
|
1578
|
+
drags through `onComponentChange`; selection policies such as closing active transform
|
|
1579
|
+
controls belong to the parent editor, not to `HierarchyPanel` or the standalone panels.
|
|
1580
|
+
|
|
1581
|
+
### Reusable XR scene editor debug UI
|
|
1582
|
+
|
|
1583
|
+
For temporary client-app scene setup, prefer the reusable scene editor adapters
|
|
1584
|
+
from `@vincentt-xr/sdk/debug-ui` instead of rewriting viewport, hierarchy, and
|
|
1585
|
+
inspector wiring in each app. These are debug/authoring adapters over the
|
|
1586
|
+
scene-object runtime, not production runtime UI.
|
|
1587
|
+
|
|
1588
|
+
```tsx
|
|
1589
|
+
import {
|
|
1590
|
+
SceneHierarchyPanel,
|
|
1591
|
+
SceneInspectorPanel,
|
|
1592
|
+
ViewportToolbar,
|
|
1593
|
+
XRSceneEditorProvider,
|
|
1594
|
+
XRSceneViewport,
|
|
1595
|
+
} from "@vincentt-xr/sdk/debug-ui";
|
|
1596
|
+
import {
|
|
1597
|
+
replaceSceneComponent,
|
|
1598
|
+
replaceSceneObject,
|
|
1599
|
+
type RenderGroupSceneObjectStore,
|
|
1600
|
+
} from "@vincentt-xr/sdk/scene-object";
|
|
1601
|
+
|
|
1602
|
+
<XRSceneEditorProvider
|
|
1603
|
+
store={store}
|
|
1604
|
+
selectedObjectId={selectedObjectId}
|
|
1605
|
+
onSelectObject={setSelectedObjectId}
|
|
1606
|
+
onStoreChange={setStore}
|
|
1607
|
+
onComponentChange={(component) =>
|
|
1608
|
+
setStore((current) => replaceSceneComponent(current, component))
|
|
1609
|
+
}
|
|
1610
|
+
onObjectChange={(object) => setStore((current) => replaceSceneObject(current, object))}
|
|
1611
|
+
>
|
|
1612
|
+
<XRSceneViewport />
|
|
1613
|
+
<ViewportToolbar />
|
|
1614
|
+
<SceneHierarchyPanel />
|
|
1615
|
+
<SceneInspectorPanel />
|
|
1616
|
+
</XRSceneEditorProvider>;
|
|
1617
|
+
```
|
|
1618
|
+
|
|
1619
|
+
All pieces are optional and composable:
|
|
1620
|
+
|
|
1621
|
+
- `XRSceneViewport` renders `XRScene`, an editor-owned camera-texture plane,
|
|
1622
|
+
editor grid, camera, orbit controls, and `SceneObjectRenderer` entries from
|
|
1623
|
+
`createRenderPlan`. The editor video plane consumes the same camera texture as
|
|
1624
|
+
`VideoBackground`, but it is a separate debug-ui plane with `depthTest` and
|
|
1625
|
+
`depthWrite` enabled. Do not add depth props to core `VideoBackground`.
|
|
1626
|
+
- `SceneHierarchyPanel` adapts `HierarchyPanel` to a
|
|
1627
|
+
`RenderGroupSceneObjectStore`.
|
|
1628
|
+
- `SceneInspectorPanel` renders component settings for the selected object and
|
|
1629
|
+
routes edits through `onComponentChange` or `onObjectChange`.
|
|
1630
|
+
- `ViewportToolbar` owns transient viewport controls such as orbit on/off.
|
|
1631
|
+
|
|
1632
|
+
Keep `SceneObjectStore`/`RenderGroupSceneObjectStore` as the only persistent
|
|
1633
|
+
scene data. Selection, orbit state, grid visibility, video-background visibility,
|
|
1634
|
+
and editor camera choices are transient editor session state. Do not keep
|
|
1635
|
+
page-local shadow copies of Transform, MeshRenderer, Light, LookAt, or PinToMesh
|
|
1636
|
+
settings.
|
|
1637
|
+
|
|
1638
|
+
Use these adapters in development/debug surfaces such as `/viewport-editor`
|
|
1639
|
+
or a temporary client-app editor. Omit them for final preview/runtime and render
|
|
1640
|
+
the scene through core runtime components instead.
|
|
1641
|
+
|
|
1642
|
+
For debug pages that need a camera/image/video picker, import
|
|
1643
|
+
`MediaSourceSwitcher` and `useFaceTrackingMediaSourceSession` from
|
|
1644
|
+
`@vincentt-xr/sdk/debug-ui`. `useFaceTrackingMediaSourceSession(activeMediaSource)`
|
|
1645
|
+
applies the selected debug media preset to the current XR session and starts
|
|
1646
|
+
the face-tracking media pipeline used by the debug face examples. The older
|
|
1647
|
+
`useMediaSourceSession(activeMediaSource, { pipeline })` name remains available
|
|
1648
|
+
for custom debug pipelines. Use these only in debug/example UI; production
|
|
1649
|
+
runtime should set the session media source directly.
|
|
1650
|
+
|
|
1651
|
+
### Debug UI usage in Harness client apps
|
|
1652
|
+
|
|
1653
|
+
#### AI-agent integration contract
|
|
1654
|
+
|
|
1655
|
+
Choose the import door by responsibility. Do not search other doors for an
|
|
1656
|
+
equivalent symbol and do not import SDK source files directly.
|
|
1657
|
+
|
|
1658
|
+
| Agent task | Import door | Symbols |
|
|
1659
|
+
| ----------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------ |
|
|
1660
|
+
| Render managed objects and compose passes | `@vincentt-xr/sdk` | `SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer` |
|
|
1661
|
+
| Guard internal render layers | `@vincentt-xr/sdk` | `MAX_RENDER_GROUP_PASSES`, `getRenderGroupLayer`, `EXTERNAL_BACKGROUND_RENDER_LAYER` |
|
|
1662
|
+
| Create and mutate scene-object state | `@vincentt-xr/sdk/scene-object` | `createSceneObject`, `createDefaultComponent`, store helpers |
|
|
1663
|
+
| Materialize and order render groups | `@vincentt-xr/sdk/scene-object` | `migrateToRenderGroups`, `createRenderPlan`, reorder/move helpers |
|
|
1664
|
+
| Render DOM authoring controls | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`, settings panels |
|
|
1665
|
+
| Type a bulk hierarchy export | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanelCopyConfig` |
|
|
1666
|
+
|
|
1667
|
+
The integrating client, not the SDK panels, must own all mutable editor state:
|
|
1668
|
+
|
|
1669
|
+
- the complete scene store, including `objects`, `components`, `renderGroups`,
|
|
1670
|
+
and `renderGroupOrder`;
|
|
1671
|
+
- `selectedObjectId` and which selected Inspector is rendered;
|
|
1672
|
+
- component replacement and object/group visibility updates;
|
|
1673
|
+
- reorder and compatibility-checked move operations;
|
|
1674
|
+
- the rule that changing selection closes transform controls and guides belonging
|
|
1675
|
+
to the previously selected object; and
|
|
1676
|
+
- the `HierarchyPanelCopyConfig` value.
|
|
1677
|
+
|
|
1678
|
+
Required invariants:
|
|
1679
|
+
|
|
1680
|
+
- Mount DOM debug panels outside `XRScene`; mount R3F/Three.js renderers and
|
|
1681
|
+
controls inside `XRScene`.
|
|
1682
|
+
- Keep both branches below the same parent state and inside `XRProvider`.
|
|
1683
|
+
- Derive group order from `scene.renderGroupOrder`; never hardcode a canonical
|
|
1684
|
+
order such as background/general/tracking/foreground/post.
|
|
1685
|
+
- Use `SequentialRenderPasses` for guaranteed cross-group composition. Do not
|
|
1686
|
+
replace it with a larger Three.js `renderOrder` value.
|
|
1687
|
+
- Treat camera layers as internal visibility masks, not user-facing layer order.
|
|
1688
|
+
- Keep each managed object in exactly one materialized group and exactly once in
|
|
1689
|
+
that group's `objectIds`.
|
|
1690
|
+
- Do not show transform controls for an unselected object. Initial control state
|
|
1691
|
+
must agree with the initial `selectedObjectId`.
|
|
1692
|
+
- Do not create more than `MAX_RENDER_GROUP_PASSES` managed groups.
|
|
1693
|
+
- Treat Copy Config as a versioned export/debug payload, not as an import or
|
|
1694
|
+
persistence API.
|
|
1695
|
+
|
|
1696
|
+
Use these state transitions:
|
|
1697
|
+
|
|
1698
|
+
| Event | Required parent-state update |
|
|
1699
|
+
| ----------------- | -------------------------------------------------------------------------------- |
|
|
1700
|
+
| Select object | Set `selectedObjectId`; disable all existing 2D guides and 3D transform controls |
|
|
1701
|
+
| Edit component | Replace that component immutably in `scene.components` |
|
|
1702
|
+
| Reorder group | Call `reorderRenderGroup`; keep object lists unchanged |
|
|
1703
|
+
| Reorder object | Call `reorderObjectWithinGroup`; change only the selected group's `objectIds` |
|
|
1704
|
+
| Move object | Validate compatibility, then call `moveObjectToGroup` |
|
|
1705
|
+
| Toggle visibility | Update the object or group; do not encode visibility in selection state |
|
|
1706
|
+
| Render frame | Derive `createRenderPlan(scene)` and active pass layers from current state |
|
|
1707
|
+
|
|
1708
|
+
Failure recovery for generated integrations:
|
|
1709
|
+
|
|
1710
|
+
| Symptom | Likely cause | Required fix | Do not |
|
|
1711
|
+
| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- |
|
|
1712
|
+
| “R3F hooks can only be used within Canvas” | R3F renderer/control mounted in the DOM branch, or duplicate React/R3F packages | Move the scene component inside `XRScene`; in link mode pin React, React DOM, Three.js, R3F, and Drei to client copies | Move DOM panels into `XRScene` |
|
|
1713
|
+
| Hierarchy order changes but visual composition does not | Only `renderOrder` or array order changed | Recompute the render plan and pass the ordered layers to `SequentialRenderPasses` | Expose camera-layer numbers as visual order |
|
|
1714
|
+
| Wrong object's gizmo remains visible | Selection changed without clearing component control flags | Disable all stale transform controls/guides in the same selection transition | Assume `HierarchyPanel` owns control state |
|
|
1715
|
+
| Screen transform drag stops after one move | An unstable callback caused active pointer listeners to be replaced | Use current SDK components and keep component replacement callbacks wired to parent state | Fork private drag logic into the DOM panel |
|
|
1716
|
+
| Cross-group drop silently fails | Target policy rejects the object's component composition | Call `validateGroupCompatibility` and preserve the original store on `false` | Force incompatible membership |
|
|
1717
|
+
| Group is omitted from the render plan | Its zero-based index is `>= MAX_RENDER_GROUP_PASSES` (`30`), so the 31st group onward is beyond the managed-pass limit | Prevent creation using `MAX_RENDER_GROUP_PASSES` | Reuse/clamp multiple groups onto one layer |
|
|
1718
|
+
| Copied JSON cannot be restored | Copy Config has no public importer | Treat it as export/debug handoff only | Invent an undocumented restore API |
|
|
1719
|
+
|
|
1720
|
+
Harness clients must keep the DOM authoring UI and R3F scene integration in sibling
|
|
1721
|
+
branches under the same parent-owned state:
|
|
1722
|
+
|
|
1723
|
+
```text
|
|
1724
|
+
XRProvider
|
|
1725
|
+
└─ AspectRatioContainer
|
|
1726
|
+
├─ XRScene (R3F Canvas)
|
|
1727
|
+
│ ├─ camera, lights, and scene content
|
|
1728
|
+
│ ├─ SequentialRenderPasses
|
|
1729
|
+
│ ├─ SceneObjectRenderer instances
|
|
1730
|
+
│ └─ transform guides and controls
|
|
1731
|
+
├─ HierarchyPanel
|
|
1732
|
+
└─ selected-object Inspector or standalone settings panel
|
|
1733
|
+
```
|
|
1734
|
+
|
|
1735
|
+
Mount `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`,
|
|
1736
|
+
`ScreenTextSettingsPanel`, and `ScreenImageSettingsPanel` outside `XRScene`.
|
|
1737
|
+
They render DOM portals and communicate through props and callbacks. Mount
|
|
1738
|
+
`SceneObjectRenderer`, `SequentialRenderPasses`, `ScreenSpaceUI`, `Transform3D`,
|
|
1739
|
+
and all Three.js content inside `XRScene` because they use R3F Canvas context.
|
|
1740
|
+
Keep both branches inside `XRProvider`.
|
|
1741
|
+
|
|
1742
|
+
The parent Harness component owns:
|
|
1743
|
+
|
|
1744
|
+
- the complete scene-object store;
|
|
1745
|
+
- `selectedObjectId`;
|
|
1746
|
+
- component replacement;
|
|
1747
|
+
- group and object reorder callbacks;
|
|
1748
|
+
- visibility callbacks;
|
|
1749
|
+
- compatibility-checked moves; and
|
|
1750
|
+
- the bulk Copy Config value.
|
|
1751
|
+
|
|
1752
|
+
The data loop is:
|
|
1753
|
+
|
|
1754
|
+
```text
|
|
1755
|
+
Hierarchy / Inspector callback
|
|
1756
|
+
↓
|
|
1757
|
+
parent scene store
|
|
1758
|
+
↓
|
|
1759
|
+
selection + render plan + active pass layers
|
|
1760
|
+
↓
|
|
1761
|
+
SceneObjectRenderer inside XRScene
|
|
1762
|
+
↓
|
|
1763
|
+
transform-control change
|
|
1764
|
+
└──────────────→ parent scene store
|
|
1765
|
+
```
|
|
1766
|
+
|
|
1767
|
+
If React reports that R3F hooks can only be used within Canvas, a scene renderer,
|
|
1768
|
+
render-pass controller, transform guide, or Three.js control has crossed into the
|
|
1769
|
+
DOM panel branch. Do not solve this by moving the panels into `XRScene`; move the
|
|
1770
|
+
R3F-dependent scene component back inside `XRScene` and keep the shared state above
|
|
1771
|
+
both branches.
|
|
1772
|
+
|
|
1773
|
+
The working `test001` Harness client is the canonical external integration. It runs
|
|
1774
|
+
a Cube, Screen Image, and Screen Text together with `HierarchyPanel`,
|
|
1775
|
+
selection-driven Inspectors, component editors, group/object ordering, visibility,
|
|
1776
|
+
transform controls, sequential render passes, and bulk Copy Config. The SDK
|
|
1777
|
+
`/layer-panel-test` page is the matching in-repo fixture.
|
|
1778
|
+
|
|
1779
|
+
For SDK co-development, run `pnpm build:watch` in `vincentt-xr-sdk` and
|
|
1780
|
+
`SDK_LINK=1 pnpm dev` in the Harness client. Use a different `PORT` when the SDK
|
|
1781
|
+
example is already using 5173, for example
|
|
1782
|
+
`PORT=5174 SDK_LINK=1 pnpm dev`. Also validate the client typecheck and linked
|
|
1783
|
+
production build; the in-repo SDK example alone does not validate package exports
|
|
1784
|
+
or Canvas placement in a separate client.
|
|
1785
|
+
|
|
1786
|
+
### Render groups and hierarchy ordering
|
|
1787
|
+
|
|
1788
|
+
Import render-group types, policies, migration, compatibility, reorder helpers, and
|
|
1789
|
+
`createRenderPlan` from `@vincentt-xr/sdk/scene-object`. Import
|
|
1790
|
+
`SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer`,
|
|
1791
|
+
`getRenderGroupLayer`, and `EXTERNAL_BACKGROUND_RENDER_LAYER` from core.
|
|
1792
|
+
|
|
1793
|
+
Use `migrateToRenderGroups(store, objectOrder)` to group a flat scene-object store.
|
|
1794
|
+
It preserves compatible `renderGroupId` assignments, infers missing or incompatible
|
|
1795
|
+
assignments from components, and creates only the groups required by actual scene
|
|
1796
|
+
objects. Their initial order follows the first occurrence in `objectOrder`; there is
|
|
1797
|
+
no fixed render-order list. New
|
|
1798
|
+
objects can call `getDefaultRenderGroupId(components)` after their components are
|
|
1799
|
+
assembled.
|
|
1800
|
+
|
|
1801
|
+
The ownership boundary is:
|
|
1802
|
+
|
|
1803
|
+
| Concern | Owner |
|
|
1804
|
+
| -------------------------------------------------- | -------------------------- |
|
|
1805
|
+
| Object identity, display name, and component order | `SceneObject` |
|
|
1806
|
+
| Object behavior and settings | `SceneComponent` |
|
|
1807
|
+
| Cross-domain composition and group visibility | `RenderGroup` |
|
|
1808
|
+
| Order inside a group | `RenderGroup.objectIds` |
|
|
1809
|
+
| Order between groups | `renderGroupOrder` |
|
|
1810
|
+
| Structural commands | `HierarchyPanel` callbacks |
|
|
1811
|
+
| Property editing and selection policy | Parent editor + Inspector |
|
|
1812
|
+
| Object-to-pass mapping | `createRenderPlan()` |
|
|
1813
|
+
| Final frame composition | `SequentialRenderPasses` |
|
|
1814
|
+
|
|
1815
|
+
The Hierarchy and Inspector emit commands but do not own scene state or rendering.
|
|
1816
|
+
Every managed object must belong to exactly one materialized group, appear exactly
|
|
1817
|
+
once across all group `objectIds` arrays, and have a matching `renderGroupId`.
|
|
1818
|
+
Selection is editor state and never changes render order.
|
|
1819
|
+
|
|
1820
|
+
Use only the immutable helpers to change structure:
|
|
1821
|
+
|
|
1822
|
+
- `reorderRenderGroup(store, groupId, nextIndex)` changes group composition order.
|
|
1823
|
+
- `reorderObjectWithinGroup(store, groupId, objectId, nextIndex)` sorts within a group.
|
|
1824
|
+
- `moveObjectToGroup(...)` moves only when `validateGroupCompatibility(...)` passes.
|
|
1825
|
+
- `createRenderPlan(store)` returns top-first entries and stable 1000-value sorting bands.
|
|
1826
|
+
|
|
1827
|
+
Unknown group/object IDs, incompatible targets, and reorders to the current index
|
|
1828
|
+
are safe no-ops that return the original store. `validateGroupCompatibility` is
|
|
1829
|
+
read-only and returns `false` for missing objects, missing groups, or rejected
|
|
1830
|
+
component compositions.
|
|
1831
|
+
|
|
1832
|
+
Pass each plan entry's `renderOrder` and `renderLayer` to `SceneObjectRenderer`.
|
|
1833
|
+
Mount one `SequentialRenderPasses` with the active plan layers in top-first order.
|
|
1834
|
+
It takes over the R3F loop, preserves color between groups, and clears depth before
|
|
1835
|
+
each later group. Bottom groups therefore composite in front even across Three.js's
|
|
1836
|
+
opaque and transparent queues. `captureFrame()` uses the same grouped renderer.
|
|
1837
|
+
|
|
1838
|
+
These mechanisms are different:
|
|
1839
|
+
|
|
1840
|
+
- `renderGroupOrder` controls cross-group sequential composition. The first group
|
|
1841
|
+
renders first; the last group renders last and appears in front.
|
|
1842
|
+
- `RenderGroup.objectIds` controls order inside one group.
|
|
1843
|
+
- Three.js camera layers are internal visibility masks; their numeric values do not
|
|
1844
|
+
define visual order.
|
|
1845
|
+
- `renderOrder` is a sorting value inside one render call. It cannot by itself
|
|
1846
|
+
impose ordering across opaque and transparent queues.
|
|
1847
|
+
|
|
1848
|
+
```tsx
|
|
1849
|
+
const plan = createRenderPlan(scene);
|
|
1850
|
+
const passLayers = scene.renderGroupOrder.flatMap((groupId, groupIndex) =>
|
|
1851
|
+
scene.renderGroups[groupId]?.enabled && scene.renderGroups[groupId].objectIds.length
|
|
1852
|
+
? [getRenderGroupLayer(groupIndex)]
|
|
1853
|
+
: [],
|
|
1854
|
+
);
|
|
1855
|
+
|
|
1856
|
+
<SequentialRenderPasses layers={passLayers} />;
|
|
1857
|
+
{
|
|
1858
|
+
plan.map((entry) => (
|
|
1859
|
+
<SceneObjectRenderer
|
|
1860
|
+
key={entry.objectId}
|
|
1861
|
+
objectId={entry.objectId}
|
|
1862
|
+
store={scene}
|
|
1863
|
+
renderOrder={entry.renderOrder}
|
|
1864
|
+
renderLayer={entry.renderLayer}
|
|
1865
|
+
/>
|
|
1866
|
+
));
|
|
1867
|
+
}
|
|
1868
|
+
```
|
|
1869
|
+
|
|
1870
|
+
Layer 0 is shared across passes for lights and non-renderable scene infrastructure.
|
|
1871
|
+
All visible managed project content in a grouped scene must be assigned to a group
|
|
1872
|
+
layer. `SequentialRenderPasses` also enables its layers on the R3F interaction
|
|
1873
|
+
raycaster. `RenderGroupLayer` leaves Drei `TransformControls` on shared layer 0,
|
|
1874
|
+
so 2D guides and 3D move/rotate/scale gizmos remain draggable after render-group
|
|
1875
|
+
isolation.
|
|
1876
|
+
|
|
1877
|
+
For legacy flat camera backgrounds, render `VideoBackground` outside managed
|
|
1878
|
+
scene-object content. For the reusable scene editor, use `XRSceneViewport`; it
|
|
1879
|
+
owns a separate depth-writing camera-texture plane for editor-only 3D placement.
|
|
1880
|
+
|
|
1881
|
+
`HierarchyPanel` receives ordered `groups`, scene `objects`, selection callbacks,
|
|
1882
|
+
and separate callbacks for group reorder, within-group reorder, and explicit moves.
|
|
1883
|
+
It owns collapse UI and drop feedback, but the parent store owns selection, group
|
|
1884
|
+
visibility, compatibility validation, the Inspector, and rendering.
|
|
1885
|
+
Empty reserved groups are not materialized in the store or presented in the hierarchy.
|
|
1886
|
+
The panel is expanded by default and can fold to its title button without losing
|
|
1887
|
+
its position, current selection, or per-group collapse state. The normal hierarchy
|
|
1888
|
+
shows human-readable names only; internal domains, IDs, camera layers, and ordering
|
|
1889
|
+
values belong in deliberate debug tooling rather than the primary object list.
|
|
1890
|
+
Pass a `HierarchyPanelCopyConfig` value as `copyConfig` to show one bulk
|
|
1891
|
+
**Copy Config** action. The panel writes that value as formatted JSON and confirms
|
|
1892
|
+
with **Copied Config**. For a complete scene
|
|
1893
|
+
handoff, include `renderGroupOrder`, `renderGroups`, `objects`, and `components`;
|
|
1894
|
+
omit transient editor state such as current selection and panel position. The copied
|
|
1895
|
+
group order therefore contains only groups backed by current scene objects.
|
|
1896
|
+
|
|
1897
|
+
Use this versioned shape for a complete handoff:
|
|
1898
|
+
|
|
1899
|
+
```ts
|
|
1900
|
+
type SceneConfigV1 = {
|
|
1901
|
+
kind: "scene";
|
|
1902
|
+
version: 1;
|
|
1903
|
+
renderGroupOrder: RenderGroupId[];
|
|
1904
|
+
renderGroups: Record<RenderGroupId, RenderGroup>;
|
|
1905
|
+
objects: Record<SceneObjectId, SceneObject>;
|
|
1906
|
+
components: Record<SceneComponentId, SceneComponent>;
|
|
1907
|
+
};
|
|
1908
|
+
```
|
|
1909
|
+
|
|
1910
|
+
This is currently an export/debugging contract, not a guaranteed round-trip
|
|
1911
|
+
format: there is no public scene-config importer. Exclude selection, panel
|
|
1912
|
+
position, fold/collapse state, active drag state, feedback messages, and any
|
|
1913
|
+
flat external `VideoBackground`.
|
|
1914
|
+
|
|
1915
|
+
Current compatibility and limits:
|
|
1916
|
+
|
|
1917
|
+
| Domain | Required composition | Status |
|
|
1918
|
+
| ------------------ | -------------------------------------------------- | --------- |
|
|
1919
|
+
| General / 3D Scene | `Transform` + `MeshRenderer` | supported |
|
|
1920
|
+
| 2D Foreground | `ScreenTransform` + `ScreenText` or `ScreenImage` | supported |
|
|
1921
|
+
| Background | editor camera-texture plane or external background | partial |
|
|
1922
|
+
| AR Tracking | tracking scene-object components | reserved |
|
|
1923
|
+
| Post Effects | post-processing scene object | reserved |
|
|
1924
|
+
|
|
1925
|
+
Reserved groups are not materialized or displayed until compatible managed scene
|
|
1926
|
+
objects exist. Each active group adds one scene render, with a maximum of
|
|
1927
|
+
`MAX_RENDER_GROUP_PASSES` (currently 30) managed passes. Import that constant from
|
|
1928
|
+
`@vincentt-xr/sdk` when guarding group creation. `createRenderPlan` omits groups
|
|
1929
|
+
beyond the limit instead of throwing or assigning colliding camera layers.
|
|
1930
|
+
`EXTERNAL_BACKGROUND_RENDER_LAYER` remains temporary infrastructure for legacy
|
|
1931
|
+
external background composition. The reusable scene editor should prefer its
|
|
1932
|
+
own camera-texture plane instead of treating `VideoBackground` as a managed
|
|
1933
|
+
scene object.
|
|
1934
|
+
|
|
1935
|
+
An explicit `renderGroupId` assignment is retained only when its policy accepts the
|
|
1936
|
+
object's components; otherwise grouping uses the inferred compatible domain. Object
|
|
1937
|
+
IDs, component IDs, and component settings remain unchanged.
|
|
1938
|
+
|
|
1939
|
+
The render-group tests protect derived group order, compatible defaults, local
|
|
1940
|
+
object reorder, rejected move no-ops, stable render bands/layers, sequential depth
|
|
1941
|
+
clearing, grouped capture, and interaction across active camera layers. The
|
|
1942
|
+
compiling reference implementation is `/layer-panel-test` in the SDK example.
|
|
1943
|
+
|
|
1944
|
+
### Shared debug authoring surface for `ScreenText`
|
|
1945
|
+
|
|
1946
|
+
The SDK also ships a dev-only inspector at `lib/debug-ui/screen-text-settings`
|
|
1947
|
+
for `ScreenText` authoring. It groups the text renderer, transform, style, and
|
|
1948
|
+
background controls into separate sections so creators can inspect the lane
|
|
1949
|
+
without shipping the debug chrome in production.
|
|
1950
|
+
|
|
1951
|
+
The panel's transform copy button copies a JSON payload for the selected object
|
|
1952
|
+
including `name`, `kind: "screenText"`, `transform`, `content`, `style`,
|
|
1953
|
+
`layout`, and `screenSpaceUI`. The `transform` includes `coordinateSpace` and
|
|
1954
|
+
`canvasSize`. Use that payload as the handoff from human layout tuning back to
|
|
1955
|
+
code.
|
|
1956
|
+
|
|
1957
|
+
### Legacy debug layer panel
|
|
1958
|
+
|
|
1959
|
+
The SDK ships a reusable `LayerPanel` in `lib/debug-ui/layer-panel` for
|
|
1960
|
+
flat-stack compatibility. New grouped editors should use `HierarchyPanel`.
|
|
1961
|
+
Both are intended for dev / creator tooling only. Production shipping surfaces
|
|
1962
|
+
should not include either panel.
|
|
1963
|
+
|
|
1964
|
+
---
|
|
1965
|
+
|
|
376
1966
|
## `<TextLabel>` — Canvas-rendered text panel
|
|
377
1967
|
|
|
378
1968
|
Import from `@vincentt-xr/sdk`. Billboarded in 3D space.
|
|
379
1969
|
|
|
380
|
-
| prop
|
|
381
|
-
|
|
|
382
|
-
| `text`
|
|
383
|
-
| `
|
|
384
|
-
| `
|
|
385
|
-
| `
|
|
386
|
-
| `
|
|
387
|
-
| `
|
|
388
|
-
| `
|
|
389
|
-
| `
|
|
390
|
-
| `
|
|
391
|
-
| `
|
|
1970
|
+
| prop | type | default | notes |
|
|
1971
|
+
| --------------- | ------------------------------------------------- | ---------------------- | --------------------------------------------------------------------------------------------- |
|
|
1972
|
+
| `text` | `string` | **required** | The text to render. |
|
|
1973
|
+
| `lines` | `string[]` | — | Optional precomputed lines. Skips automatic word wrapping. |
|
|
1974
|
+
| `color` | `string` | `#ffffff` | Text color. |
|
|
1975
|
+
| `strokeLayers` | `{ color, width, opacity? }[]` | — | Optional multi-stroke stack, drawn outer-to-inner. |
|
|
1976
|
+
| `strokeColor` | `string` | — | Optional text stroke color. Pair with `strokeWidth`. |
|
|
1977
|
+
| `strokeWidth` | `number` | `0` | Text stroke thickness in canvas px. |
|
|
1978
|
+
| `shadowLayers` | `{ color, blur, offsetX?, offsetY?, opacity? }[]` | — | Optional multi-shadow/glow stack, drawn back-to-front. |
|
|
1979
|
+
| `shadowColor` | `string` | — | Optional text shadow / glow color. |
|
|
1980
|
+
| `shadowBlur` | `number` | `0` | Shadow blur radius in canvas px. |
|
|
1981
|
+
| `shadowOffsetX` | `number` | `0` | Shadow X offset in canvas px. |
|
|
1982
|
+
| `shadowOffsetY` | `number` | `0` | Shadow Y offset in canvas px. |
|
|
1983
|
+
| `fontSize` | `number` | `48` | Canvas px at the default 512px canvas — NOT world units. Chips/badges typically need 200-320. |
|
|
1984
|
+
| `textAlign` | `"left" \| "center" \| "right"` | `"center"` | Horizontal text alignment inside the available lane. |
|
|
1985
|
+
| `overflow` | `"clip" \| "clamp" \| "ellipsis" \| "shrink"` | — | Overflow behavior. `clamp` keeps full visible lines, `clip` allows canvas-edge clipping. |
|
|
1986
|
+
| `resizeToFit` | `boolean` | — | Keep shrinking until the text fits or `minFontSize` is reached. |
|
|
1987
|
+
| `minFontSize` | `number` | `12` | Minimum shrink target in canvas px. |
|
|
1988
|
+
| `bgColor` | `string` | `rgba(20,40,110,0.88)` | Panel background; use `"transparent"` for text-only. |
|
|
1989
|
+
| `bgFit` | `"full" \| "text"` | `"full"` | Background fit mode. Use `"text"` for pill/capsule backgrounds that hug the text block. |
|
|
1990
|
+
| `borderColor` | `string` | — | Border color; omit for no border. |
|
|
1991
|
+
| `borderRadius` | `number` | `24` | Corner radius; 0 for square. |
|
|
1992
|
+
| `aspect` | `number` | `2.5` | Panel shape (width ÷ height). >1 = wide/banner, <1 = tall. |
|
|
1993
|
+
| `scale` | `[x,y,z] \| number` | `1` | World size. Combine with `aspect` to make wide banners. |
|
|
1994
|
+
| `position` | `[x,y,z]` | `[0,0,0]` | World position. +Y up, -Y down. |
|
|
1995
|
+
| `name` | `string` | — | Set on every element you create. |
|
|
392
1996
|
|
|
393
1997
|
**Gotchas:**
|
|
394
1998
|
|
|
395
1999
|
- `fontSize` is canvas px, not world units. Chips/badges need 200-320, not 16-48.
|
|
396
2000
|
- `fontSize` does not change panel shape — use `aspect` (shape) and `scale` (size).
|
|
397
|
-
- `
|
|
2001
|
+
- `textAlign` affects both glyph placement and `bgFit="text"` background placement.
|
|
2002
|
+
- `TextLabel` respects the `visible` prop.
|
|
2003
|
+
- For text-first 2D treatments, prefer `strokeColor` + `strokeWidth` and `shadowColor` + `shadowBlur` before stacking multiple `TextLabel`s. Native text effects keep the glyph edges aligned and resize cleanly.
|
|
2004
|
+
- Use `strokeLayers` / `shadowLayers` when you need richer looks like sticker text, double-outline, or offset-drop-shadow. The single `stroke*` / `shadow*` props are shorthand for one layer and can coexist with the arrays.
|
|
2005
|
+
- `TextLabel` now owns real overflow behavior. If you need exact authored line breaks, pass `text` with `\n` or provide `lines` directly; those lines will be rendered as the source layout instead of being re-wrapped from one flattened string.
|
|
2006
|
+
- Overflow semantics:
|
|
2007
|
+
- `clip`: draw all resolved lines and let the canvas bounds crop them, including partial last lines.
|
|
2008
|
+
- `clamp`: keep only full visible lines within the line budget.
|
|
2009
|
+
- `ellipsis`: same as clamp, but ellipsize the last visible line.
|
|
2010
|
+
- `shrink`: reduce font size until the content fits or `minFontSize` is reached, then clamp if it still overflows.
|
|
398
2011
|
- **Inside a `<ScreenTransform>`, the parent stretches the label to fill the anchor rect — `aspect` must equal the rect's aspect or the banner distorts and leaves dead space around the text.** The rect's aspect is `(right − left) / (top − bottom)`. So for a full-width top banner `anchors={{ left: -1, right: 1, top: 1, bottom: 0.76 }}` → aspect `= 2 / 0.24 ≈ 8.3`. Set `borderRadius={0}` for a true edge-to-edge banner. (The rect controls size + placement — don't add `scale`/`position` to fill it; set the anchors instead.)
|
|
2012
|
+
- If you want the SDK to own that anchor math, use `<ScreenText>` instead of pairing `<ScreenTransform>` with `<TextLabel>` manually.
|
|
399
2013
|
|
|
400
2014
|
```tsx
|
|
401
2015
|
// Full-width banner pinned to the top, INSIDE a ScreenTransform.
|
|
@@ -445,21 +2059,28 @@ import { AspectRatioContainer } from "@vincentt-xr/sdk";
|
|
|
445
2059
|
|
|
446
2060
|
Grab a photo of the live render or record the canvas, then save or share it.
|
|
447
2061
|
|
|
448
|
-
### `
|
|
2062
|
+
### `useFrameCapture()` + `dataURLtoFile()` — Single-shot photo
|
|
449
2063
|
|
|
450
|
-
`
|
|
451
|
-
|
|
452
|
-
|
|
2064
|
+
Use `useFrameCapture` for UI/effect logic. It exposes `status`, `capturePhoto`,
|
|
2065
|
+
`clear`, `dataUrl`, and `error`, plus `onCaptureStart`, `onCaptureComplete`, and
|
|
2066
|
+
`onError`. Status is `idle | capturing | completed | error`. The underlying
|
|
2067
|
+
capture forces a fresh render and returns a `data:` URL; `dataURLtoFile` turns
|
|
2068
|
+
it into a `File` for upload or the Web Share API. `capturePhoto()` resolves to
|
|
2069
|
+
the URL on success or `null` after setting `status: "error"` and `error` on
|
|
2070
|
+
failure; callers do not need a rejection handler.
|
|
453
2071
|
|
|
454
2072
|
```tsx
|
|
455
|
-
import { dataURLtoFile } from "@vincentt-xr/sdk";
|
|
456
|
-
import { useXRContext } from "@vincentt-xr/sdk/low-level";
|
|
2073
|
+
import { dataURLtoFile, useFrameCapture } from "@vincentt-xr/sdk";
|
|
457
2074
|
|
|
458
|
-
const
|
|
459
|
-
|
|
460
|
-
|
|
2075
|
+
const photo = useFrameCapture({
|
|
2076
|
+
onCaptureComplete: (dataUrl) => share(dataURLtoFile(dataUrl, "photo.png")),
|
|
2077
|
+
});
|
|
2078
|
+
await photo.capturePhoto();
|
|
461
2079
|
```
|
|
462
2080
|
|
|
2081
|
+
For synchronous low-level code, `session.captureFrame()` remains available via
|
|
2082
|
+
`useXRContext` from `@vincentt-xr/sdk/low-level`.
|
|
2083
|
+
|
|
463
2084
|
### `useMediaRecorder()` — Record the canvas to a video Blob
|
|
464
2085
|
|
|
465
2086
|
Records a `MediaStream` (from the canvas's `captureStream()`) to a `Blob`. Picks
|
|
@@ -476,6 +2097,20 @@ await rec.start(canvas.captureStream(30));
|
|
|
476
2097
|
rec.stop(); // -> rec.recordedBlob is the video Blob
|
|
477
2098
|
```
|
|
478
2099
|
|
|
2100
|
+
Use `rec.status` as the source of truth:
|
|
2101
|
+
`idle | starting | recording | paused | stopping | completed | error`.
|
|
2102
|
+
Controls are `start`, `pause`, `resume`, `stop`, and `clear`. Events are
|
|
2103
|
+
`onRecordingStart`, `onRecordingPause`, `onRecordingResume`,
|
|
2104
|
+
`onRecordingStop`, `onRecordingComplete`, and `onError`. Keep `isRecording`
|
|
2105
|
+
and `isPaused` only as convenience booleans; do not invent parallel state.
|
|
2106
|
+
When `passThroughStream` is configured, call `start()` without an argument;
|
|
2107
|
+
otherwise call `start(stream)` with the canvas capture stream.
|
|
2108
|
+
The verified `/capture-recording` example exercises the complete photo and video
|
|
2109
|
+
lifecycle against the live XR canvas. It displays a recording HUD and elapsed
|
|
2110
|
+
time for `starting`, `recording`, `paused`, `stopping`, and `completed`, then
|
|
2111
|
+
previews the resulting `recordedBlob` in a video player. Photo and recording
|
|
2112
|
+
flows were verified with a real webcam on 26 July 2026.
|
|
2113
|
+
|
|
479
2114
|
---
|
|
480
2115
|
|
|
481
2116
|
## Audio
|
|
@@ -535,3 +2170,154 @@ function SoundToggle() {
|
|
|
535
2170
|
Audio assets: m4a (AAC). A rare app needing a second, independent mixer can
|
|
536
2171
|
`new AudioEngine()` and drive it directly; the free functions always target the
|
|
537
2172
|
one default engine.
|
|
2173
|
+
|
|
2174
|
+
## 3D behavior components: LookAt and PinToMesh
|
|
2175
|
+
|
|
2176
|
+
`LookAt` and `PinToMesh` are partial-stage XRScene behavior components. They are
|
|
2177
|
+
exported for editor/API prototyping, but they are not yet a fully verified
|
|
2178
|
+
Effect House parity surface.
|
|
2179
|
+
|
|
2180
|
+
`LookAt` rotates its child toward a target object resolved either from a direct
|
|
2181
|
+
`Object3D` prop or from serialized scene-object target ids. Treat it as suitable
|
|
2182
|
+
for current editor experiments, not final production behavior, until point vs
|
|
2183
|
+
direction mode, axis conventions, offset ordering, and missing-target behavior
|
|
2184
|
+
are covered by tests.
|
|
2185
|
+
|
|
2186
|
+
When `SceneObjectRenderer` renders nested objects, child scene objects do not
|
|
2187
|
+
inherit the parent's LookAt reorientation by default. `LookAtSettings` includes
|
|
2188
|
+
`inheritChildren: boolean` for the special case where children should rigidly
|
|
2189
|
+
follow the parent's look-at facing. Defaults from
|
|
2190
|
+
`createDefaultComponent("LookAt", objectId)` set `inheritChildren: false`.
|
|
2191
|
+
|
|
2192
|
+
`PinToMesh` wraps a source object and pins that source wrapper to a target mesh
|
|
2193
|
+
surface. Pass `PinToMeshSettings` with
|
|
2194
|
+
`enabled: true` and a non-null `targetObjectId`; default settings are disabled
|
|
2195
|
+
with no target. The component supports static mesh surface lookup, UV input,
|
|
2196
|
+
orientation, vertex-normal option, offset space, offset position, and offset
|
|
2197
|
+
rotation. Offset position and offset rotation default to `{ x: 0, y: 0, z: 0 }`.
|
|
2198
|
+
When PinToMesh is disabled or the target becomes unavailable, the runtime pin
|
|
2199
|
+
wrapper resets to local position `0,0,0` and identity rotation so the child
|
|
2200
|
+
`Transform3D` remains the authored source of truth.
|
|
2201
|
+
|
|
2202
|
+
Pin wrapper rotation is identity-based for `orientation: "none"` and
|
|
2203
|
+
`orientation: "position"`. `orientation: "positionAndDirection"` aligns the
|
|
2204
|
+
wrapper to the target surface normal before offset rotation is applied. Offset
|
|
2205
|
+
rotation is reapplied from a clean base each frame and must not compound over
|
|
2206
|
+
time.
|
|
2207
|
+
|
|
2208
|
+
Use `PinToMeshSettingsPanel` from `@vincentt-xr/sdk/debug-ui` for editor
|
|
2209
|
+
controls. The panel uses Transform-style text number inputs for UV and offsets:
|
|
2210
|
+
visible zero defaults, decimal validation, 0.1 arrow-step increments, Shift for
|
|
2211
|
+
larger steps, and Alt for finer steps. Serialized scenes should store stable
|
|
2212
|
+
scene-object ids for targets; runtime renderers resolve those ids to registered
|
|
2213
|
+
`Object3D` instances.
|
|
2214
|
+
|
|
2215
|
+
Direct runtime usage:
|
|
2216
|
+
|
|
2217
|
+
```tsx
|
|
2218
|
+
import { LookAt, PinToMesh, Transform3D, MeshRenderer } from "@vincentt-xr/sdk";
|
|
2219
|
+
|
|
2220
|
+
<PinToMesh
|
|
2221
|
+
target={targetObject}
|
|
2222
|
+
settings={{
|
|
2223
|
+
enabled: true,
|
|
2224
|
+
targetObjectId: "base-plane",
|
|
2225
|
+
orientation: "positionAndDirection",
|
|
2226
|
+
uv: { x: 0.5, y: 0.5 },
|
|
2227
|
+
vertexNormal: true,
|
|
2228
|
+
offsetSpace: "local",
|
|
2229
|
+
offsetPosition: { x: 0, y: 0.25, z: 0 },
|
|
2230
|
+
offsetRotation: { x: 0, y: 0, z: 0 },
|
|
2231
|
+
}}
|
|
2232
|
+
>
|
|
2233
|
+
<Transform3D value={transformSettings}>
|
|
2234
|
+
<LookAt
|
|
2235
|
+
target={lookTargetObject}
|
|
2236
|
+
settings={{
|
|
2237
|
+
enabled: true,
|
|
2238
|
+
targetObjectId: "target-object",
|
|
2239
|
+
mode: "point",
|
|
2240
|
+
offsetRotation: { x: 0, y: 0, z: 0 },
|
|
2241
|
+
}}
|
|
2242
|
+
>
|
|
2243
|
+
<MeshRenderer value={meshSettings} />
|
|
2244
|
+
</LookAt>
|
|
2245
|
+
</Transform3D>
|
|
2246
|
+
</PinToMesh>;
|
|
2247
|
+
```
|
|
2248
|
+
|
|
2249
|
+
Serialized component defaults:
|
|
2250
|
+
|
|
2251
|
+
- `createDefaultComponent("LookAt", objectId)` returns disabled LookAt settings
|
|
2252
|
+
with `targetObjectId: null`, `mode: "point"`, and zero offset rotation.
|
|
2253
|
+
- `createDefaultComponent("PinToMesh", objectId)` returns disabled PinToMesh
|
|
2254
|
+
settings with `targetObjectId: null`, UV `{ x: 0.5, y: 0.5 }`,
|
|
2255
|
+
`orientation: "none"`, `vertexNormal: false`, `offsetSpace: "local"`, and
|
|
2256
|
+
zero position/rotation offsets.
|
|
2257
|
+
|
|
2258
|
+
Current limitations:
|
|
2259
|
+
|
|
2260
|
+
- UV behavior is not final. Missing UVs currently fall back to target bounds
|
|
2261
|
+
center behavior instead of a visible invalid-target state.
|
|
2262
|
+
- LookAt has a minimal API: `enabled`, `targetObjectId`, `mode`, and
|
|
2263
|
+
`offsetRotation`. Axis remapping and world-up controls are still planned.
|
|
2264
|
+
- Runtime target resolution is scoped by `XRProvider` through an internal
|
|
2265
|
+
runtime-object registry. Direct calls to `registerRuntimeObject` and
|
|
2266
|
+
`resolveRuntimeObject` outside an `XRProvider` use a legacy fallback registry
|
|
2267
|
+
and should be avoided for multi-scene pages.
|
|
2268
|
+
|
|
2269
|
+
## 3D light scene objects
|
|
2270
|
+
|
|
2271
|
+
`LightObject` renders a serialized light component through `Transform3D`.
|
|
2272
|
+
Import from `@vincentt-xr/sdk`.
|
|
2273
|
+
|
|
2274
|
+
```tsx
|
|
2275
|
+
import { LightObject } from "@vincentt-xr/sdk";
|
|
2276
|
+
|
|
2277
|
+
<LightObject
|
|
2278
|
+
transform={transformSettings}
|
|
2279
|
+
settings={{
|
|
2280
|
+
enabled: true,
|
|
2281
|
+
type: "directional",
|
|
2282
|
+
color: "#ffffff",
|
|
2283
|
+
intensity: 2.2,
|
|
2284
|
+
castShadow: true,
|
|
2285
|
+
distance: 10,
|
|
2286
|
+
angle: Math.PI / 6,
|
|
2287
|
+
penumbra: 0.2,
|
|
2288
|
+
decay: 2,
|
|
2289
|
+
showHelper: false,
|
|
2290
|
+
}}
|
|
2291
|
+
/>;
|
|
2292
|
+
```
|
|
2293
|
+
|
|
2294
|
+
`LightSettings.type` is `"ambient" | "directional" | "point" | "spot"`.
|
|
2295
|
+
Directional, point, and spot lights support `castShadow`; ambient lights only
|
|
2296
|
+
contribute global light. `showHelper` draws a small non-physical editor marker
|
|
2297
|
+
for the light object and defaults to `false`; enable it only in editor/debug
|
|
2298
|
+
surfaces.
|
|
2299
|
+
|
|
2300
|
+
For an Effect House-style declarative player, use `<AudioPlayer>`. It supports
|
|
2301
|
+
`once`, a finite `loop` count, and `infinite` playback, plus autoplay intent,
|
|
2302
|
+
per-player mute, and a clamped 0–100 volume. Browser autoplay policy still
|
|
2303
|
+
applies: audible autoplay may reject until a user gesture, surfaced through
|
|
2304
|
+
`onError`. The component is headless (it renders a hidden `<audio>`); attach a
|
|
2305
|
+
ref for custom controls.
|
|
2306
|
+
|
|
2307
|
+
```tsx
|
|
2308
|
+
import { AudioPlayer, type AudioPlayerHandle } from "@vincentt-xr/sdk";
|
|
2309
|
+
|
|
2310
|
+
const player = useRef<AudioPlayerHandle>(null);
|
|
2311
|
+
|
|
2312
|
+
<AudioPlayer
|
|
2313
|
+
ref={player}
|
|
2314
|
+
src={musicUrl}
|
|
2315
|
+
playMode="loop"
|
|
2316
|
+
loopCount={3}
|
|
2317
|
+
autoplay
|
|
2318
|
+
volume={80}
|
|
2319
|
+
onError={(error) => console.warn(error)}
|
|
2320
|
+
/>;
|
|
2321
|
+
|
|
2322
|
+
// From a user gesture: player.current?.play(), .pause(), or .stop().
|
|
2323
|
+
```
|