@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GROUNDING.md +5651 -82
- package/README.md +1 -0
- package/dist/CanonicalHeadFrame-BHcWyaiK.js +1506 -0
- package/dist/CanonicalHeadFrame-D-_T1OMU.cjs +1 -0
- package/dist/DebuggerUI-7ngnJsBe.js +275 -0
- package/dist/DebuggerUI-CoglgKdO.cjs +1 -0
- package/dist/FaceRetouch-CLoTfrcs.js +1071 -0
- package/dist/FaceRetouch-J1lC1WbS.cjs +134 -0
- package/dist/FaceSurface-CHAMEeG1.cjs +1 -0
- package/dist/FaceSurface-DmP9AEeC.js +27 -0
- package/dist/SegmentationOverlay-D_FWkcyP.cjs +35 -0
- package/dist/SegmentationOverlay-vn5x_knv.js +6237 -0
- package/dist/SkeletonUtils-C2r90MtN.js +21 -0
- package/dist/SkeletonUtils-CddIup1F.cjs +1 -0
- package/dist/TextLabel-XE0rQSDn.cjs +2 -0
- package/dist/TextLabel-c9BCucOI.js +961 -0
- package/dist/assets/cache.d.ts +99 -0
- package/dist/assets/estimate.d.ts +11 -0
- package/dist/assets/gltfLoader.d.ts +16 -0
- package/dist/assets/index.d.ts +7 -0
- package/dist/assets/loaders.d.ts +43 -0
- package/dist/assets/react.d.ts +33 -0
- package/dist/body-pose/index.d.ts +521 -0
- package/dist/body-trigger/__fixtures__/syntheticBody.d.ts +83 -0
- package/dist/body-trigger/frame.d.ts +88 -0
- package/dist/body-trigger/index.d.ts +6 -0
- package/dist/body-trigger/moves.d.ts +235 -0
- package/dist/body-trigger/trigger.d.ts +75 -0
- package/dist/calibration/calibration.d.ts +75 -0
- package/dist/calibration/index.d.ts +6 -0
- package/dist/calibration/storage.d.ts +13 -0
- package/dist/calibration/types.d.ts +53 -0
- package/dist/calibration/useXRCalibration.d.ts +46 -0
- package/dist/camera/CameraPermissionGate.d.ts +26 -0
- package/dist/camera/devices.d.ts +26 -0
- package/dist/camera/index.d.ts +8 -0
- package/dist/camera/permission.d.ts +86 -0
- package/dist/camera/useCameraPermission.d.ts +35 -0
- package/dist/canonicalFace-BHwyCx1c.js +9 -0
- package/dist/canonicalFace-C83I225R.cjs +1 -0
- package/dist/capabilities/detect.d.ts +45 -0
- package/dist/capabilities/index.d.ts +5 -0
- package/dist/capabilities/types.d.ts +120 -0
- package/dist/capabilities/useXRCapabilities.d.ts +18 -0
- package/dist/compat/environment.d.ts +5 -0
- package/dist/compat/index.d.ts +4 -0
- package/dist/compat/run.d.ts +18 -0
- package/dist/compat/scenarios.d.ts +10 -0
- package/dist/compat/types.d.ts +101 -0
- package/dist/components/BodyInstruction/BodyPoseHint.d.ts +63 -0
- package/dist/components/BodyInstruction/index.d.ts +69 -0
- package/dist/components/BodyInstruction/useBodyPose.d.ts +23 -0
- package/dist/components/BodyTracker/index.d.ts +17 -0
- package/dist/components/BodyTrigger/index.d.ts +49 -0
- package/dist/components/Debugger/DebuggerProvider.d.ts +14 -1
- package/dist/components/Debugger/DebuggerUI.d.ts +3 -1
- package/dist/components/FaceAtlas/FaceAtlasLayer.d.ts +114 -0
- package/dist/components/FaceAtlas/MadeUpCamera.d.ts +80 -0
- package/dist/components/FaceAtlas/constants.d.ts +8 -0
- package/dist/components/FaceAtlas/index.d.ts +7 -0
- package/dist/components/FaceAtlas/useFaceAtlasSurface.d.ts +84 -0
- package/dist/components/FaceAtlas/useFaceSkin.d.ts +84 -0
- package/dist/components/FaceCharacter/ExternalFaceMesh.d.ts +109 -0
- package/dist/components/FaceCharacter/FaceExpressionController.d.ts +53 -0
- package/dist/components/FaceCharacter/FaceMorph.d.ts +71 -0
- package/dist/components/FaceCharacter/FacePiercing.d.ts +65 -0
- package/dist/components/FaceCharacter/FaceReprojection.d.ts +85 -0
- package/dist/components/FaceCharacter/FaceRig.d.ts +38 -0
- package/dist/components/FaceCharacter/FaceSurface.d.ts +49 -0
- package/dist/components/FaceCharacter/FaceSurfaceOccluder.d.ts +39 -0
- package/dist/components/FaceCharacter/PinToFaceMesh.d.ts +68 -0
- package/dist/components/FaceCharacter/index.d.ts +18 -0
- package/dist/components/FaceCharacter/useFaceReprojector.d.ts +10 -0
- package/dist/components/FaceInset/FaceInset.d.ts +68 -0
- package/dist/components/FaceInset/frame.d.ts +94 -0
- package/dist/components/FaceInset/index.d.ts +6 -0
- package/dist/components/FaceInset/material.d.ts +40 -0
- package/dist/components/FaceInset/useFaceInsetTexture.d.ts +37 -0
- package/dist/components/FaceInstruction/FaceExpressionHint.d.ts +43 -0
- package/dist/components/FaceInstruction/index.d.ts +56 -0
- package/dist/components/FaceInstruction/useFaceExpression.d.ts +20 -0
- package/dist/components/FaceLiquify/FaceLiquify.d.ts +49 -0
- package/dist/components/FaceLiquify/debug.d.ts +37 -0
- package/dist/components/FaceLiquify/frame.d.ts +55 -0
- package/dist/components/FaceLiquify/index.d.ts +10 -0
- package/dist/components/FaceLiquify/material.d.ts +39 -0
- package/dist/components/FaceLiquify/pass.d.ts +49 -0
- package/dist/components/FaceLiquify/presence.d.ts +25 -0
- package/dist/components/FaceLiquify/presets.d.ts +82 -0
- package/dist/components/FaceLiquify/selfTest.d.ts +66 -0
- package/dist/components/FaceLiquify/target.d.ts +85 -0
- package/dist/components/FaceLiquify/warp.d.ts +72 -0
- package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +37 -5
- package/dist/components/FaceMesh/canonical/geometryGroups.d.ts +21 -0
- package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +21 -0
- package/dist/components/FaceMesh/canonical/shaders.d.ts +1 -1
- package/dist/components/FaceMesh/index.d.ts +2 -2
- package/dist/components/FaceRetouch/Background.d.ts +3 -1
- package/dist/components/FaceRetouch/FaceRetouch.d.ts +5 -0
- package/dist/components/FaceRetouch/renderState.d.ts +19 -0
- package/dist/components/FaceSwap/FaceMirror.d.ts +46 -0
- package/dist/components/FaceSwap/FaceSwap.d.ts +58 -0
- package/dist/components/FaceSwap/FaceTextureLayer.d.ts +72 -0
- package/dist/components/FaceSwap/faceSwapPlan.d.ts +91 -0
- package/dist/components/FaceSwap/index.d.ts +13 -0
- package/dist/components/FaceSwap/settings.d.ts +300 -0
- package/dist/components/FaceSwap/useFaceTexture.d.ts +35 -0
- package/dist/components/FaceSwap/useLiveFace.d.ts +38 -0
- package/dist/components/FaceSwap/useLiveFaceTexture.d.ts +31 -0
- package/dist/components/FaceWarp3D/FaceWarp3D.d.ts +50 -0
- package/dist/components/FaceWarp3D/index.d.ts +2 -3
- package/dist/components/Faceless/Faceless.d.ts +51 -0
- package/dist/components/Faceless/index.d.ts +2 -0
- package/dist/components/GestureListener/index.d.ts +11 -0
- package/dist/components/HairLock/HairLock.d.ts +39 -0
- package/dist/components/HairLock/index.d.ts +2 -0
- package/dist/components/HairLock/lockGeometry.d.ts +15 -0
- package/dist/components/HandCutout/index.d.ts +35 -0
- package/dist/components/HandInstruction/HandGestureHint.d.ts +70 -0
- package/dist/components/HandInstruction/index.d.ts +74 -0
- package/dist/components/HandInstruction/useHandGesture.d.ts +22 -0
- package/dist/components/HandRig/ForearmOccluder.d.ts +29 -0
- package/dist/components/HandRig/HandAttachment.d.ts +34 -0
- package/dist/components/HandRig/HandOccluder.d.ts +55 -0
- package/dist/components/HandRig/HandTattoo.d.ts +61 -0
- package/dist/components/HandRig/WristAttachment.d.ts +34 -0
- package/dist/components/HandRig/index.d.ts +101 -0
- package/dist/components/HandRig/useArmMask.d.ts +29 -0
- package/dist/components/HandRig/wristSolve.d.ts +69 -0
- package/dist/components/HandTouch/HandTouch.d.ts +62 -0
- package/dist/components/HandTouch/Touchable.d.ts +36 -0
- package/dist/components/HandTouch/index.d.ts +2 -0
- package/dist/components/HandTracker/index.d.ts +10 -0
- package/dist/components/HeadBinding/index.d.ts +14 -0
- package/dist/components/HeadOccluder/CanonicalHeadFrame.d.ts +22 -0
- package/dist/components/HeadOccluder/Earrings.d.ts +55 -0
- package/dist/components/HeadOccluder/HeadAttachment.d.ts +38 -0
- package/dist/components/HeadOccluder/NeckOccluder.d.ts +64 -0
- package/dist/components/HeadOccluder/earringPlacement.d.ts +31 -0
- package/dist/components/HeadOccluder/headAttachmentPoints.d.ts +69 -0
- package/dist/components/HeadOccluder/index.d.ts +57 -3
- package/dist/components/HeadOccluder/useCanonicalHeadFit.d.ts +36 -0
- package/dist/components/HeadOccluder/useLandmarkPoseFit.d.ts +80 -0
- package/dist/components/HolisticTracker/index.d.ts +69 -0
- package/dist/components/HolisticTracker/settings.d.ts +58 -0
- package/dist/components/HolisticTracker/signals.d.ts +49 -0
- package/dist/components/Instruction/HintStage.d.ts +29 -0
- package/dist/components/Instruction/InstructionCue.d.ts +126 -0
- package/dist/components/Instruction/gestureHold.d.ts +20 -0
- package/dist/components/Instruction/ghost.d.ts +129 -0
- package/dist/components/Instruction/hintBlend.d.ts +19 -0
- package/dist/components/Instruction/hintLook.d.ts +81 -0
- package/dist/components/Instruction/hintLoop.d.ts +36 -0
- package/dist/components/Instruction/index.d.ts +41 -0
- package/dist/components/Instruction/useGhostModel.d.ts +33 -0
- package/dist/components/Instruction/useInstructionHold.d.ts +72 -0
- package/dist/components/IrisEffect/IrisMaterial.d.ts +73 -0
- package/dist/components/IrisEffect/index.d.ts +51 -0
- package/dist/components/IrisEffect/settings.d.ts +117 -0
- package/dist/components/LipEffect/LipEffect.d.ts +27 -1
- package/dist/components/LipEffect/LipEffectMaterial.d.ts +95 -7
- package/dist/components/LipEffect/LipEffects.d.ts +12 -0
- package/dist/components/LipEffect/defaults.d.ts +46 -4
- package/dist/components/LipEffect/index.d.ts +11 -3
- package/dist/components/LipEffect/lipShading.d.ts +73 -0
- package/dist/components/LipEffect/settings.d.ts +375 -0
- package/dist/components/LipEffect/shader.d.ts +39 -0
- package/dist/components/LipEffect/types.d.ts +128 -9
- package/dist/components/LipEffect/useLipDesignTextures.d.ts +17 -0
- package/dist/components/LipEffect/uvGuide.d.ts +74 -0
- package/dist/components/Makeup/Blush.d.ts +18 -0
- package/dist/components/Makeup/Brows.d.ts +20 -0
- package/dist/components/Makeup/Contour.d.ts +17 -0
- package/dist/components/Makeup/Eyeliner.d.ts +20 -0
- package/dist/components/Makeup/Eyeshadow.d.ts +20 -0
- package/dist/components/Makeup/Highlighter.d.ts +18 -0
- package/dist/components/Makeup/MakeupDesign.d.ts +37 -0
- package/dist/components/Makeup/MakeupRegion.d.ts +132 -0
- package/dist/components/Makeup/constants.d.ts +44 -0
- package/dist/components/Makeup/design.d.ts +106 -0
- package/dist/components/Makeup/designSamples.d.ts +42 -0
- package/dist/components/Makeup/eyes.d.ts +114 -0
- package/dist/components/Makeup/fields.d.ts +10 -0
- package/dist/components/Makeup/geometry.d.ts +106 -0
- package/dist/components/Makeup/index.d.ts +26 -0
- package/dist/components/Makeup/looks.d.ts +28 -0
- package/dist/components/Makeup/material.d.ts +239 -0
- package/dist/components/Makeup/settings.d.ts +54 -0
- package/dist/components/Makeup/shades.d.ts +143 -0
- package/dist/components/Makeup/shapes.d.ts +27 -0
- package/dist/components/Makeup/skinTone.d.ts +135 -0
- package/dist/components/Makeup/types.d.ts +110 -0
- package/dist/components/Makeup/useMakeupSkinTone.d.ts +40 -0
- package/dist/components/Necklace/Necklace.d.ts +44 -0
- package/dist/components/Necklace/index.d.ts +1 -0
- package/dist/components/ParticleEmitter/ParticleEmitter.d.ts +69 -0
- package/dist/components/ParticleEmitter/index.d.ts +1 -0
- package/dist/components/PersonCutout/index.d.ts +50 -0
- package/dist/components/PersonCutout/personCutout.d.ts +16 -0
- package/dist/components/ScreenImage/components/ScreenImageBody.d.ts +1 -1
- package/dist/components/ScreenImage/index.d.ts +3 -1
- package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts +1 -1
- package/dist/components/ScreenImage/materials/ScreenImageMaskMaterial.d.ts +11 -0
- package/dist/components/ScreenImage/source.d.ts +6 -0
- package/dist/components/ScreenImage/types.d.ts +18 -0
- package/dist/components/SegmentationOverlay/SegmentationOverlay.d.ts +37 -0
- package/dist/components/SegmentationOverlay/SegmentationOverlayMaterial.d.ts +31 -0
- package/dist/components/SegmentationOverlay/index.d.ts +9 -0
- package/dist/components/SegmentationOverlay/maskTexture.d.ts +23 -0
- package/dist/components/SegmentationOverlay/providers.d.ts +72 -0
- package/dist/components/SegmentationOverlay/settings.d.ts +65 -0
- package/dist/components/ShoulderOccluder/index.d.ts +43 -0
- package/dist/components/ShoulderOccluder/useCanonicalTorsoFit.d.ts +24 -0
- package/dist/components/SpringSwing/SpringSwing.d.ts +22 -0
- package/dist/components/SpringSwing/index.d.ts +1 -0
- package/dist/components/XRProvider/index.d.ts +16 -2
- package/dist/components/XRScene/renderOutputTarget.d.ts +12 -0
- package/dist/components/XRViewport/hooks/useViewportExternalFaces.d.ts +1 -1
- package/dist/components/XRViewport/hooks/useViewportMediaSource.d.ts +1 -1
- package/dist/components/XRViewport/hooks/useViewportPipeline.d.ts +1 -1
- package/dist/components/index.d.ts +11 -0
- package/dist/config/index.d.ts +0 -1
- package/dist/constants/index.d.ts +4 -1
- package/dist/context-DTxSQvp4.js +13 -0
- package/dist/context-hA0Rb2Xa.cjs +1 -0
- package/dist/contexts-BH7PdKFS.cjs +1 -0
- package/dist/contexts-jak8-tND.js +667 -0
- package/dist/coordinates/feed.d.ts +52 -0
- package/dist/coordinates/index.d.ts +7 -0
- package/dist/coordinates/side.d.ts +40 -0
- package/dist/coordinates/spaces.d.ts +49 -0
- package/dist/coordinates/viewport.d.ts +22 -0
- package/dist/coordinates/world.d.ts +15 -0
- package/dist/debug-ui/default-setting.cjs +2 -2
- package/dist/debug-ui/face-swap-settings/index.d.ts +22 -0
- package/dist/debug-ui/gesture-settings/index.d.ts +14 -0
- package/dist/debug-ui/hierarchy-panel/index.d.ts +3 -2
- package/dist/debug-ui/holistic-tracker-settings/index.d.ts +16 -0
- package/dist/debug-ui/index.d.ts +18 -3
- package/dist/debug-ui/iris-effect-settings/index.d.ts +12 -0
- package/dist/debug-ui/lip-effect-settings/index.d.ts +15 -0
- package/dist/debug-ui/media-source/index.d.ts +1 -1
- package/dist/debug-ui/media-source/session.d.ts +30 -2
- package/dist/debug-ui/media-source.cjs +1 -1
- package/dist/debug-ui/media-source.js +1 -1
- package/dist/debug-ui/object-inspector-panel/index.d.ts +3 -2
- package/dist/debug-ui/product-hotspots/index.d.ts +25 -0
- package/dist/debug-ui/scene-editor/hierarchy-panel.d.ts +3 -1
- package/dist/debug-ui/scene-editor/inspector-panel.d.ts +3 -1
- package/dist/debug-ui/screen-image-settings/index.d.ts +3 -1
- package/dist/debug-ui/screen-text-settings/ScreenTextSettingsPanel.d.ts +1 -1
- package/dist/debug-ui/screen-text-settings/types.d.ts +2 -0
- package/dist/debug-ui/segmentation-overlay-settings/index.d.ts +12 -0
- package/dist/debug-ui/settings-panel/index.d.ts +6 -1
- package/dist/debug-ui/shared/debugWindow.d.ts +51 -0
- package/dist/debug-ui/xr-diagnostics/index.d.ts +26 -0
- package/dist/debug-ui.cjs +3 -3
- package/dist/debug-ui.js +2695 -2547
- package/dist/detector-K8aej4MB.js +59 -0
- package/dist/detector-ijypGtse.cjs +1 -0
- package/dist/entries/all.d.ts +3 -1
- package/dist/entries/core.d.ts +20 -2
- package/dist/entries/face-effects.d.ts +123 -9
- package/dist/entries/instructions.d.ts +48 -0
- package/dist/entries/internal.d.ts +2 -1
- package/dist/entries/low-level.d.ts +49 -3
- package/dist/entries/particles.d.ts +11 -0
- package/dist/entries/product.d.ts +11 -0
- package/dist/entries/tracking.d.ts +60 -2
- package/dist/face-atlas/attributes.d.ts +30 -0
- package/dist/face-atlas/eyes.d.ts +272 -0
- package/dist/face-atlas/faceAtlas.generated.d.ts +46 -0
- package/dist/face-atlas/glsl/atlasVertex.d.ts +46 -0
- package/dist/face-atlas/glsl/common.d.ts +55 -0
- package/dist/face-atlas/glsl/design.d.ts +49 -0
- package/dist/face-atlas/glsl/eyes.d.ts +73 -0
- package/dist/face-atlas/glsl/finish.d.ts +24 -0
- package/dist/face-atlas/glsl/format.d.ts +4 -0
- package/dist/face-atlas/glsl/index.d.ts +59 -0
- package/dist/face-atlas/glsl/shading.d.ts +15 -0
- package/dist/face-atlas/glsl/shapes.d.ts +66 -0
- package/dist/face-atlas/glsl/sparkle.d.ts +19 -0
- package/dist/face-atlas/glsl/uniforms.d.ts +106 -0
- package/dist/face-atlas/guide.d.ts +51 -0
- package/dist/face-atlas/index.d.ts +25 -0
- package/dist/face-atlas/lead.d.ts +117 -0
- package/dist/face-atlas/madeUpCamera.d.ts +125 -0
- package/dist/face-atlas/masks.d.ts +53 -0
- package/dist/face-atlas/obj.d.ts +69 -0
- package/dist/face-atlas/polylines.d.ts +63 -0
- package/dist/face-atlas/probe/FaceSkinProbe.d.ts +208 -0
- package/dist/face-atlas/probe/__fixtures__/emulate.d.ts +9 -0
- package/dist/face-atlas/probe/__fixtures__/face.d.ts +24 -0
- package/dist/face-atlas/probe/constants.d.ts +137 -0
- package/dist/face-atlas/probe/glsl.d.ts +43 -0
- package/dist/face-atlas/probe/lightFit.d.ts +108 -0
- package/dist/face-atlas/probe/schedule.d.ts +44 -0
- package/dist/face-atlas/probe/selfTest.d.ts +87 -0
- package/dist/face-atlas/probe/stats.d.ts +143 -0
- package/dist/face-atlas/probe/targets.d.ts +70 -0
- package/dist/face-atlas/probe/unwrap.d.ts +76 -0
- package/dist/face-atlas/quality.d.ts +28 -0
- package/dist/face-atlas/regions.d.ts +31 -0
- package/dist/face-atlas/registry.d.ts +252 -0
- package/dist/face-atlas/renderState.d.ts +86 -0
- package/dist/face-atlas/selfOcclusion.d.ts +62 -0
- package/dist/face-atlas/shading.d.ts +406 -0
- package/dist/face-atlas/shapes.d.ts +86 -0
- package/dist/face-atlas/sides.d.ts +41 -0
- package/dist/face-atlas/skinMask.d.ts +35 -0
- package/dist/face-atlas/surface.d.ts +155 -0
- package/dist/face-atlas/topology.d.ts +44 -0
- package/dist/face-character/__fixtures__/faceImageDetections.d.ts +25 -0
- package/dist/face-character/__fixtures__/faceNode.d.ts +22 -0
- package/dist/face-character/bindFaceModel.d.ts +106 -0
- package/dist/face-character/canonicalFace.d.ts +17 -0
- package/dist/face-character/canonicalFaceGeometry.d.ts +11 -0
- package/dist/face-character/canonicalFacePositions.d.ts +7 -0
- package/dist/face-character/driveFaceModel.d.ts +57 -0
- package/dist/face-character/expressionOutputs.d.ts +45 -0
- package/dist/face-character/faceExpressions.d.ts +104 -0
- package/dist/face-character/faceImageLandmarker.d.ts +93 -0
- package/dist/face-character/faceMeshBinding.d.ts +62 -0
- package/dist/face-character/faceMorph.d.ts +77 -0
- package/dist/face-character/faceMorphs.d.ts +46 -0
- package/dist/face-character/facePhoto.d.ts +138 -0
- package/dist/face-character/facePiercings.d.ts +116 -0
- package/dist/face-character/facePin.d.ts +131 -0
- package/dist/face-character/faceReprojection.d.ts +273 -0
- package/dist/face-character/faceReprojector.d.ts +171 -0
- package/dist/face-character/faceRig.d.ts +176 -0
- package/dist/face-character/faceSwapPresence.d.ts +85 -0
- package/dist/face-character/faceTexture.d.ts +169 -0
- package/dist/face-character/faceTextureBaker.d.ts +86 -0
- package/dist/face-character/faceTextureGuide.d.ts +106 -0
- package/dist/face-character/faceTextureStats.d.ts +298 -0
- package/dist/face-character/faceTransfer.d.ts +396 -0
- package/dist/face-character/faceTransferGeometry.d.ts +109 -0
- package/dist/face-character/faceTransferSelfTest.d.ts +94 -0
- package/dist/face-character/faceWarp3D.d.ts +101 -0
- package/dist/face-character/faceless.d.ts +79 -0
- package/dist/face-character/featureWarp.d.ts +28 -0
- package/dist/face-character/liveFaceRegistry.d.ts +370 -0
- package/dist/face-character/liveValue.d.ts +12 -0
- package/dist/face-character/memo.d.ts +15 -0
- package/dist/face-effects.cjs +2324 -1
- package/dist/face-effects.js +14208 -2268
- package/dist/face-expression/index.d.ts +182 -0
- package/dist/face-info/landmarks.d.ts +17 -3
- package/dist/face-info/types.d.ts +20 -3
- package/dist/gestures/definitions.d.ts +98 -0
- package/dist/gestures/detector.d.ts +87 -0
- package/dist/gestures/index.d.ts +8 -0
- package/dist/gestures/measures.d.ts +114 -0
- package/dist/gestures/settings.d.ts +49 -0
- package/dist/getFaceInfo-B1otSDMX.cjs +1 -0
- package/dist/getFaceInfo-Cu4xDWiL.js +62 -0
- package/dist/hand/armFit.d.ts +46 -0
- package/dist/hand/armMask.d.ts +178 -0
- package/dist/hand/handCutout.d.ts +23 -0
- package/dist/hand/handCutoutRegistry.d.ts +7 -0
- package/dist/hand/handGestures.d.ts +524 -0
- package/dist/hand/handPose.d.ts +98 -0
- package/dist/hand/handRigSolver.d.ts +159 -0
- package/dist/hand/handTattoo.d.ts +81 -0
- package/dist/hand/handTouch.d.ts +240 -0
- package/dist/hand/wrist.d.ts +44 -0
- package/dist/hand/wristFilter.d.ts +44 -0
- package/dist/hooks/selectors.d.ts +76 -1
- package/dist/hooks/useCalibrationFaceTracker.d.ts +1 -1
- package/dist/hooks/useFaceInfo.d.ts +13 -1
- package/dist/hooks/useFaceInfos.d.ts +1 -1
- package/dist/hooks/useFaceModelAcquisition.d.ts +38 -0
- package/dist/hooks/useGesture.d.ts +32 -0
- package/dist/hooks/useIrisTracking.d.ts +45 -0
- package/dist/hooks/useTrackerContext.d.ts +6 -0
- package/dist/hooks/useXRDiagnostics.d.ts +81 -0
- package/dist/hooks/useXRModel.d.ts +7 -0
- package/dist/hooks/useXRSession/facade.d.ts +42 -2
- package/dist/hooks/useXRSession/index.d.ts +2 -2
- package/dist/hooks/useXRSession/reason.d.ts +3 -1
- package/dist/hooks/useXRSession/registry.d.ts +14 -0
- package/dist/hooks/useXRSession/store.d.ts +52 -3
- package/dist/index-B1_TzJaI.js +1104 -0
- package/dist/index-BavF06C6.js +190 -0
- package/dist/index-BfFmpddl.cjs +21 -0
- package/dist/index-BuBzneeW.cjs +46 -0
- package/dist/index-CGsQWodV.cjs +1 -0
- package/dist/index-CIe7eCMH.js +3200 -0
- package/dist/index-CfSF8LXa.cjs +1 -0
- package/dist/index-D5GTzOzD.cjs +1 -0
- package/dist/index-DBhQzvVa.js +314 -0
- package/dist/index-DE17y0ZI.cjs +1 -0
- package/dist/index-JII7huPk.js +442 -0
- package/dist/index-tpCr9CLj.cjs +1 -0
- package/dist/index-w1S7fI2B.js +1432 -0
- package/dist/index-zGaqrz_K.js +188 -0
- package/dist/instructions.cjs +30 -0
- package/dist/instructions.js +1507 -0
- package/dist/{internal-Dmk1cTXq.js → internal-B90P-clC.js} +138 -119
- package/dist/internal-Cjinca64.cjs +1 -0
- package/dist/internal.cjs +1 -1
- package/dist/internal.js +15 -10
- package/dist/iris/compute.d.ts +27 -0
- package/dist/iris/index.d.ts +4 -0
- package/dist/iris/lead.d.ts +19 -0
- package/dist/iris/types.d.ts +89 -0
- package/dist/landmarkMask-BFDyBnKM.js +224 -0
- package/dist/landmarkMask-u7es2k4X.cjs +1 -0
- package/dist/landmarks-BlUqf5MZ.cjs +1 -0
- package/dist/landmarks-bftdwq6Z.js +76 -0
- package/dist/loaders-BGAUACsd.js +363 -0
- package/dist/loaders-BJ1_QCHt.cjs +1 -0
- package/dist/low-level.cjs +1 -1
- package/dist/low-level.js +1190 -675
- package/dist/main.cjs +1 -1
- package/dist/main.js +1190 -452
- package/dist/models/Base/index.d.ts +49 -0
- package/dist/models/Body/index.d.ts +3 -1
- package/dist/models/Face/constants.d.ts +8 -0
- package/dist/models/Face/index.d.ts +4 -12
- package/dist/models/Face/rotation.d.ts +18 -3
- package/dist/models/Hand/GestureTracking.d.ts +1 -1
- package/dist/models/Hand/HandTracking.d.ts +17 -1
- package/dist/models/Holistic/index.d.ts +49 -0
- package/dist/models/Holistic/result.d.ts +68 -0
- package/dist/models/Segmentation/index.d.ts +2 -0
- package/dist/models/SemanticSegmentation/index.d.ts +31 -0
- package/dist/models/SemanticSegmentation/presets.d.ts +28 -0
- package/dist/models/index.d.ts +2 -0
- package/dist/models/nodeSource.d.ts +9 -0
- package/dist/particles/emission.d.ts +24 -0
- package/dist/particles/eyes.d.ts +24 -0
- package/dist/particles/glsl.d.ts +14 -0
- package/dist/particles/index.d.ts +10 -0
- package/dist/particles/motion.d.ts +42 -0
- package/dist/particles/presets.d.ts +12 -0
- package/dist/particles/ring.d.ts +58 -0
- package/dist/particles/selfTest.d.ts +53 -0
- package/dist/particles/settings.d.ts +127 -0
- package/dist/particles/shapes.d.ts +29 -0
- package/dist/particles/spawner.d.ts +60 -0
- package/dist/particles/system.d.ts +64 -0
- package/dist/particles.cjs +122 -0
- package/dist/particles.js +774 -0
- package/dist/product/ProductDimensions.d.ts +16 -0
- package/dist/product/ProductHotspot.d.ts +16 -0
- package/dist/product/ProductViewer.d.ts +9 -0
- package/dist/product/details.d.ts +53 -0
- package/dist/product/index.d.ts +14 -0
- package/dist/product/loadModelViewer.d.ts +4 -0
- package/dist/product/types.d.ts +71 -0
- package/dist/product/useProductAr.d.ts +37 -0
- package/dist/product/useProductDetails.d.ts +60 -0
- package/dist/product.cjs +1 -0
- package/dist/product.js +414 -0
- package/dist/providers-C8rE26Uf.js +366 -0
- package/dist/providers-DHsrhDI2.cjs +62 -0
- package/dist/quality/XRQualityController.d.ts +25 -0
- package/dist/quality/controller.d.ts +33 -0
- package/dist/quality/index.d.ts +6 -0
- package/dist/quality/types.d.ts +100 -0
- package/dist/quality/useXRQuality.d.ts +28 -0
- package/dist/render/renderPasses.d.ts +8 -0
- package/dist/render/sceneBeforeRender.d.ts +12 -0
- package/dist/{renderGroups-KZk6eRiY.js → renderGroups-B38pUaX2.js} +1 -1
- package/dist/{renderGroups-80H3DpTl.cjs → renderGroups-Cb_4Ar1Q.cjs} +1 -1
- package/dist/renderPasses-B6l8IpJv.js +1290 -0
- package/dist/renderPasses-dqmkft4P.cjs +158 -0
- package/dist/renderState-CJNmj5jz.js +39 -0
- package/dist/renderState-apSfg4eR.cjs +1 -0
- package/dist/scene-object.cjs +1 -1
- package/dist/scene-object.js +2 -2
- package/dist/segmentation-Cb3ouK6Z.js +1758 -0
- package/dist/segmentation-D9pYO6aR.cjs +1 -0
- package/dist/segmentation2d/faceMemory.d.ts +28 -0
- package/dist/segmentation2d/faceRegions.d.ts +24 -0
- package/dist/segmentation2d/index.d.ts +9 -0
- package/dist/segmentation2d/maskUv.d.ts +23 -0
- package/dist/segmentation2d/rasterize.d.ts +14 -0
- package/dist/segmentation2d/types.d.ts +52 -0
- package/dist/select-Ck5LGz-J.cjs +1 -0
- package/dist/select-pYRo_win.js +233 -0
- package/dist/selectors-C4X71aoD.cjs +1 -0
- package/dist/selectors-DmNaKIwE.js +196 -0
- package/dist/session-BrH8G7iJ.cjs +31 -0
- package/dist/session-DgA_z6O1.js +418 -0
- package/dist/settings-BJ7FMBsO.js +111 -0
- package/dist/settings-C63FMBlF.cjs +1 -0
- package/dist/similarityFit-BFqHND2e.cjs +1 -0
- package/dist/similarityFit-Bbz-R8AY.js +91 -0
- package/dist/spring/index.d.ts +2 -0
- package/dist/spring/spring.d.ts +69 -0
- package/dist/spring/swing.d.ts +54 -0
- package/dist/subjects/index.d.ts +2 -0
- package/dist/subjects/select.d.ts +62 -0
- package/dist/subjects/types.d.ts +55 -0
- package/dist/tracking-config/apply.d.ts +57 -0
- package/dist/tracking-config/index.d.ts +7 -0
- package/dist/tracking-config/resolve.d.ts +25 -0
- package/dist/tracking-config/storage.d.ts +8 -0
- package/dist/tracking-config/types.d.ts +106 -0
- package/dist/tracking-config/useXRTrackingConfig.d.ts +38 -0
- package/dist/tracking.cjs +10 -1
- package/dist/tracking.js +1326 -506
- package/dist/trigger-D0Pm9noT.cjs +49 -0
- package/dist/trigger-IQeys0oS.js +2128 -0
- package/dist/types/compositor.d.ts +21 -0
- package/dist/types/face.d.ts +20 -1
- package/dist/types/hand-tracking.d.ts +9 -0
- package/dist/types/holistic.d.ts +121 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/model.d.ts +22 -2
- package/dist/types/nodes.d.ts +47 -3
- package/dist/types/segmentation.d.ts +24 -0
- package/dist/types/session.d.ts +41 -3
- package/dist/updateThrottle-BseecYag.js +4 -0
- package/dist/updateThrottle-Of5Y4SVa.cjs +1 -0
- package/dist/useFaceInfo-BHDNCvsJ.cjs +1 -0
- package/dist/useFaceInfo-J4x7LsqL.js +76 -0
- package/dist/useFaceResultReader-BvEo5REG.js +61 -0
- package/dist/useFaceResultReader-DuaL1KJG.cjs +1 -0
- package/dist/useInstructionHold-DOu-RpCM.js +53 -0
- package/dist/useInstructionHold-DwJY1Alr.cjs +1 -0
- package/dist/useLandmarkPoseFit-Bg-h9BPb.cjs +1 -0
- package/dist/useLandmarkPoseFit-Bmb3h6Mw.js +87 -0
- package/dist/useXRCapabilities-Cr3RzuAW.cjs +1 -0
- package/dist/useXRCapabilities-Cxmlmpbd.js +30 -0
- package/dist/utils/canonicalRigidLandmarks.d.ts +8 -0
- package/dist/utils/fns.d.ts +6 -3
- package/dist/utils/headOccluderFit.d.ts +42 -0
- package/dist/utils/integrity.d.ts +21 -1
- package/dist/utils/resolveNumFaces.d.ts +13 -0
- package/dist/utils/shoulderOccluderFit.d.ts +38 -0
- package/dist/utils/similarityFit.d.ts +23 -0
- package/dist/utils/updateThrottle.d.ts +2 -0
- package/dist/utils/useStableUniforms.d.ts +18 -0
- package/dist/uv-BMFt19bg.cjs +1 -0
- package/dist/{index-ByoNj0QP.js → uv-DJU_XfmN.js} +15 -247
- package/docs/SDK_DOCUMENTATION.md +1 -2
- package/docs/api/adaptive-quality.mdx +104 -0
- package/docs/api/asset-lifecycle.mdx +78 -0
- package/docs/api/body-instructions.mdx +285 -0
- package/docs/api/body-triggers.mdx +234 -0
- package/docs/api/camera-permission.mdx +142 -0
- package/docs/api/coordinate-spaces.mdx +168 -0
- package/docs/api/cross-camera.mdx +89 -0
- package/docs/api/debug-ui-scene-editor.mdx +23 -2
- package/docs/api/diagnostics-calibration.mdx +97 -0
- package/docs/api/external-face-mesh.mdx +231 -0
- package/docs/api/face-actions.mdx +20 -16
- package/docs/api/face-attribute-expression.mdx +1 -1
- package/docs/api/face-binding-anchors.mdx +4 -2
- package/docs/api/face-character.mdx +157 -0
- package/docs/api/face-effects.mdx +9 -0
- package/docs/api/face-expression-controller.mdx +153 -0
- package/docs/api/face-info.mdx +17 -11
- package/docs/api/face-inset.mdx +204 -0
- package/docs/api/face-instructions.mdx +183 -0
- package/docs/api/face-liquify.mdx +196 -0
- package/docs/api/face-mesh.mdx +8 -9
- package/docs/api/face-morph.mdx +117 -0
- package/docs/api/face-painting.mdx +7 -0
- package/docs/api/face-reprojection.mdx +154 -0
- package/docs/api/face-retouching.mdx +18 -10
- package/docs/api/face-swap.mdx +350 -0
- package/docs/api/face-warp-3d.mdx +166 -0
- package/docs/api/faceless.mdx +143 -0
- package/docs/api/gestures.mdx +209 -0
- package/docs/api/hair.mdx +97 -0
- package/docs/api/hand-instructions.mdx +222 -0
- package/docs/api/hand-rig.mdx +535 -0
- package/docs/api/hand-touch.mdx +162 -0
- package/docs/api/head-occluder.mdx +563 -0
- package/docs/api/holistic-tracker.mdx +164 -0
- package/docs/api/instructions.mdx +323 -0
- package/docs/api/iris-tracking.mdx +247 -0
- package/docs/api/made-up-camera.mdx +131 -0
- package/docs/api/makeup-skin-tone.mdx +135 -0
- package/docs/api/makeup.mdx +353 -0
- package/docs/api/multi-subject.mdx +98 -0
- package/docs/api/particles.mdx +155 -0
- package/docs/api/person-cutout.mdx +136 -0
- package/docs/api/pin-to-face-mesh.mdx +274 -0
- package/docs/api/product-ar.mdx +114 -0
- package/docs/api/product-details.mdx +78 -0
- package/docs/api/screen-space-coordinates.mdx +4 -1
- package/docs/api/segmentation-overlay.mdx +301 -0
- package/docs/api/session-lifecycle.mdx +136 -0
- package/docs/api/trackers.mdx +5 -0
- package/docs/api/tracking-config.mdx +108 -0
- package/docs/docs.json +51 -1
- package/docs/examples/lip-shaders.mdx +273 -0
- package/docs/guides/face-atlas-template.svg +1 -0
- package/docs/guides/face-atlas-textures.mdx +186 -0
- package/docs/guides/face-texture-template.svg +1 -0
- package/docs/guides/harness-debug-ui.mdx +4 -4
- package/docs/guides/lip-effect-textures.mdx +368 -0
- package/docs/guides/lip-uv-template.png +0 -0
- package/docs/guides/lip-uv-template.svg +1 -0
- package/docs/guides/makeup-design-textures.mdx +109 -0
- package/docs/introduction.mdx +1 -2
- package/package.json +23 -1
- package/dist/DebuggerUI-B6bP6P9r.cjs +0 -1
- package/dist/DebuggerUI-BjOMr8q_.js +0 -269
- package/dist/FaceMeshRenderer-C5l6VLdO.cjs +0 -155
- package/dist/FaceMeshRenderer-wjAHrJzP.js +0 -1270
- package/dist/FaceRetouch-BkJzoA79.cjs +0 -134
- package/dist/FaceRetouch-DwbKQJAN.js +0 -551
- package/dist/components/FaceWarp3D/FaceWarp3DHead.d.ts +0 -15
- package/dist/components/FaceWarp3D/defaults.d.ts +0 -37
- package/dist/components/FaceWarp3D/warp.d.ts +0 -13
- package/dist/contexts-ByhzppYl.js +0 -663
- package/dist/contexts-Dr62It_K.cjs +0 -1
- package/dist/debug-ui/face-warp-3d/index.d.ts +0 -6
- package/dist/getFaceInfo-C4QpCQIY.js +0 -118
- package/dist/getFaceInfo-fA1bFBub.cjs +0 -1
- package/dist/index-9dlAQAhy.js +0 -1462
- package/dist/index-BjmTe9eM.cjs +0 -1
- package/dist/index-BpEf2BBb.cjs +0 -1
- package/dist/index-CyUUurQT.cjs +0 -1
- package/dist/index-DCiMbxC3.cjs +0 -1
- package/dist/index-DXT3WccE.js +0 -2089
- package/dist/index-DpI-qo5X.cjs +0 -17
- package/dist/index-DyVF4vEW.js +0 -111
- package/dist/index-VJ0SJwmG.js +0 -214
- package/dist/internal-Cq-TEQ5Y.cjs +0 -1
- package/dist/renderLayers-lDtL0_43.cjs +0 -1
- package/dist/renderLayers-zrK2pv5O.js +0 -13
- package/dist/renderPasses-BOoBSr8z.cjs +0 -21
- package/dist/renderPasses-BcquyVtu.js +0 -1500
- package/dist/selectors-B8H97CKA.js +0 -59
- package/dist/selectors-CZLKT2Iv.cjs +0 -1
- package/dist/session-CpkiK5YA.cjs +0 -31
- package/dist/session-DLFN_VII.js +0 -363
- package/dist/useFaceInfo-BJ8-_MOs.cjs +0 -1
- package/dist/useFaceInfo-Dk426LTu.js +0 -62
- package/dist/useFaceResults-CtetsO9M.js +0 -19
- package/dist/useFaceResults-wBJF3zyp.cjs +0 -1
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Camera Permission & Privacy"
|
|
3
|
+
description: "Distinguish unknown, requesting, granted, denied, blocked and unavailable camera states; request, retry and recover without remounting; release the camera cleanly; keep every frame on the device until the viewer allows access."
|
|
4
|
+
sidebarTitle: "Camera Permission"
|
|
5
|
+
icon: "camera"
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Camera access is the first thing that can go wrong in an AR experience, and
|
|
9
|
+
the browser's `NotAllowedError` hides four different situations. The SDK
|
|
10
|
+
splits them into states a creator can act on, offers the request flow as a
|
|
11
|
+
hook and a gate component, and guarantees no camera frame is processed
|
|
12
|
+
before explicit, usable access.
|
|
13
|
+
|
|
14
|
+
## States
|
|
15
|
+
|
|
16
|
+
| State | Meaning | Next step |
|
|
17
|
+
| ------------- | ------------------------------------------------------------------------------------ | --------------------------------------------------------- |
|
|
18
|
+
| `unknown` | Never asked (the next request prompts), or the browser cannot say. | Show a primer and a request button. |
|
|
19
|
+
| `requesting` | A prompt is open. | Wait. |
|
|
20
|
+
| `granted` | Usable. | Mount the session. |
|
|
21
|
+
| `denied` | This prompt was declined or dismissed; the next request prompts again. | Offer "try again". |
|
|
22
|
+
| `blocked` | Persistently denied in the site settings; a request fails without a prompt. | Explain how to change the setting. |
|
|
23
|
+
| `unavailable` | No camera, insecure context, unsupported browser, or held by another app (`reason`). | Fall back; `reason.recoverable` says if a retry can help. |
|
|
24
|
+
|
|
25
|
+
`denied` and `blocked` are told apart by reading the Permissions API _after_
|
|
26
|
+
the failed request: `denied` there means blocked. `reason` is an
|
|
27
|
+
`XRFallbackReason` (`camera-permission-denied`, `camera-in-use`,
|
|
28
|
+
`no-camera`, `camera-insecure-context`, `camera-unsupported`, `camera-lost`).
|
|
29
|
+
|
|
30
|
+
## Before the session: the gate
|
|
31
|
+
|
|
32
|
+
```tsx
|
|
33
|
+
import { CameraPermissionGate, useCameraPermission } from "@vincentt-xr/sdk";
|
|
34
|
+
|
|
35
|
+
// Component form: children render only while access is granted.
|
|
36
|
+
<CameraPermissionGate
|
|
37
|
+
fallback={({ state, reason, canPrompt, request }) => (
|
|
38
|
+
<Primer state={state} reason={reason} onAllow={canPrompt ? () => request() : undefined} />
|
|
39
|
+
)}
|
|
40
|
+
>
|
|
41
|
+
<XRProvider>
|
|
42
|
+
<Experience />
|
|
43
|
+
</XRProvider>
|
|
44
|
+
</CameraPermissionGate>;
|
|
45
|
+
|
|
46
|
+
// Hook form, for your own layout.
|
|
47
|
+
const { state, reason, canPrompt, usable, request, refresh } = useCameraPermission();
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Neither needs an `<XRProvider>`. On mount they only _query_ the permission
|
|
51
|
+
(no prompt); `request()` prompts, and must run from a user gesture (a
|
|
52
|
+
button) as browsers require. The probe stream is stopped at once, so the
|
|
53
|
+
camera indicator goes off again and nothing is captured: the call only
|
|
54
|
+
moves the permission. Pass `auto` to prompt on mount. Both follow the
|
|
55
|
+
site-setting changes through the Permissions API, so a viewer who allows
|
|
56
|
+
the camera in the address bar flips the gate without a reload.
|
|
57
|
+
|
|
58
|
+
The built-in fallback of `<CameraPermissionGate>` announces state changes
|
|
59
|
+
with `aria-live="polite"`, keeps the request behind a real `<button>`,
|
|
60
|
+
disables it when a prompt cannot help, and labels the region "Camera access".
|
|
61
|
+
Keep those in a custom fallback: a visible button, `aria-live` on the
|
|
62
|
+
status text, `aria-disabled` when blocked, and the explanation for
|
|
63
|
+
`blocked` next to the button rather than in a toast.
|
|
64
|
+
|
|
65
|
+
## Inside the session
|
|
66
|
+
|
|
67
|
+
The session reports its own live state through `useXRSessionCameraPermission()`
|
|
68
|
+
(`@vincentt-xr/sdk/low-level`): `requesting` while its webcam
|
|
69
|
+
`getUserMedia` is pending, `granted` once bound, the classified failure
|
|
70
|
+
otherwise, and `blocked` / `unavailable` after a mid-session revoke or a
|
|
71
|
+
lost camera. Stream and canvas sources leave it `unknown`.
|
|
72
|
+
|
|
73
|
+
```tsx
|
|
74
|
+
const { state, reason } = useXRSessionCameraPermission();
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Request, retry, recover without remounting
|
|
78
|
+
|
|
79
|
+
```tsx
|
|
80
|
+
const { session } = useXRContext();
|
|
81
|
+
const retry = async () => {
|
|
82
|
+
await session.setMediaSource({ source: XRMediaSource.WEBCAM }); // prompts again if allowed
|
|
83
|
+
await session.start();
|
|
84
|
+
};
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
A failed bind leaves the session where it was (`idle` stays idle, a running
|
|
88
|
+
session keeps its old source), so the same provider, models and scene are
|
|
89
|
+
reused. A revoke mid-session stops the session with a `camera-lost` /
|
|
90
|
+
`permission-denied` error; the same two calls recover it.
|
|
91
|
+
|
|
92
|
+
### Privacy: releasing the camera
|
|
93
|
+
|
|
94
|
+
- `session.stop()` stops every media track: the browser's camera indicator
|
|
95
|
+
goes off and no frame is read. Models stay initialized, so a later
|
|
96
|
+
`setMediaSource` + `start` is fast. Use this for an in-app "privacy mode".
|
|
97
|
+
- A revoke (`permission-denied`) or a lost device (`camera-lost`) releases
|
|
98
|
+
the tracks once and suspends every model automatically.
|
|
99
|
+
- No model runs before `session.start()`, and `start()` needs a bound
|
|
100
|
+
source, so no camera data is processed before explicit access. The gate
|
|
101
|
+
adds the second guarantee: the session is not even mounted until
|
|
102
|
+
`granted`.
|
|
103
|
+
- Frames never leave the device: inference runs in the page, and the only
|
|
104
|
+
network traffic is the one-time model and engine download.
|
|
105
|
+
|
|
106
|
+
## Choosing the camera
|
|
107
|
+
|
|
108
|
+
`setMediaSource({ source: XRMediaSource.WEBCAM, facingMode: "environment" })`
|
|
109
|
+
opens the rear camera; `"user"` (default) the front one; `deviceId` picks a
|
|
110
|
+
specific camera from `enumerateDevices()`. Binding again with a different
|
|
111
|
+
facing swaps the camera mid-session. The front camera is mirrored
|
|
112
|
+
(`sourceKind` `"webcam"`), the rear camera is drawn straight (`sourceKind`
|
|
113
|
+
`"rear-camera"`), and `mirrored: true | false` overrides that for a camera
|
|
114
|
+
whose facing is unknown. Trackers and the coordinate toolkit follow the
|
|
115
|
+
source kind, so content stays aligned either way. `useXRCameraFacing()`
|
|
116
|
+
(low-level) reports the bound facing. The full cross-camera and orientation
|
|
117
|
+
contract lands with the next sprint item.
|
|
118
|
+
|
|
119
|
+
## Low-level helpers
|
|
120
|
+
|
|
121
|
+
`queryCameraPermission()`, `requestCameraAccess({ keepStream, constraints })`,
|
|
122
|
+
`subscribeCameraPermission(listener)` and `canPromptForCamera(state)` are
|
|
123
|
+
plain functions (core door) for non-React code; `CAMERA_PERMISSION_MESSAGES`
|
|
124
|
+
holds the default copy per state. `classifyCameraFailure(reason,
|
|
125
|
+
permissionState)` and `mapDomExceptionToReason(err)` live in `/low-level`
|
|
126
|
+
next to `XRErrorReason`, for callers that classify their own
|
|
127
|
+
`getUserMedia` failure.
|
|
128
|
+
|
|
129
|
+
## Manual validation
|
|
130
|
+
|
|
131
|
+
Run `/camera`:
|
|
132
|
+
|
|
133
|
+
| Case | Steps | Expect |
|
|
134
|
+
| ------------ | ----------------------------------------- | ----------------------------------------------------------------- |
|
|
135
|
+
| First visit | load | gate `unknown`, no prompt until "Allow camera" |
|
|
136
|
+
| Allow | click Allow, accept | `requesting` → `granted`; the experience mounts and binds |
|
|
137
|
+
| Dismiss | click Allow, close the prompt | `denied`, "Try again" enabled |
|
|
138
|
+
| Block | deny with "never" / site setting | `blocked`, button disabled, settings hint shown |
|
|
139
|
+
| Unblock | allow in the address bar | gate flips to `granted` without reload |
|
|
140
|
+
| Revoke | "simulate revoke" (or revoke in settings) | session `blocked` / `unavailable`, lifecycle `stopped` |
|
|
141
|
+
| Recover | "retry camera" | prompt if needed, lifecycle `running` |
|
|
142
|
+
| Privacy mode | "privacy mode (stop camera)" | camera indicator off, lifecycle `stopped`; "retry camera" resumes |
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Coordinate Spaces"
|
|
3
|
+
description: "One toolkit for source, feed, tracker, viewport, screen and world coordinates: cover crop, zoom, mirroring, orientation, and documented conversions."
|
|
4
|
+
sidebarTitle: "Coordinate Spaces"
|
|
5
|
+
icon: "arrows-to-dot"
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Every tracking consumer meets the same six spaces. Import the conversions from
|
|
9
|
+
`@vincentt-xr/sdk`; each one performs its origin, axis, crop and mirror change
|
|
10
|
+
exactly once, so never chain a manual flip or crop after it.
|
|
11
|
+
|
|
12
|
+
## The spaces
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
source (media px) feed (0..1) viewport (CSS px)
|
|
16
|
+
┌──────────────────┐ ┌────────┐ ┌────────┐
|
|
17
|
+
│ raw video/image │ cover│ │ identity │ │
|
|
18
|
+
│ 1280 × 720 │ crop │ 720 × │ per axis │ 390 × │
|
|
19
|
+
│ never mirrored │──────▶ 1280 │◀────────────────▶│ 844 │
|
|
20
|
+
└──────────────────┘ zoom │ webcam │ │ │
|
|
21
|
+
mirror│ mirror │ └────────┘
|
|
22
|
+
└───┬────┘ │
|
|
23
|
+
│ x·2−1, 1−y·2 │ design canvas
|
|
24
|
+
▼ ▼
|
|
25
|
+
tracker (−1..1, Y up) screen (720×1280 design px, Y up)
|
|
26
|
+
│
|
|
27
|
+
│ camera ray → z = 0 plane
|
|
28
|
+
▼
|
|
29
|
+
world (three.js)
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
| Space | Origin / axes | Who produces it |
|
|
33
|
+
| ------------ | ---------------------------------------------- | -------------------------------------------------------------- |
|
|
34
|
+
| **source** | top-left, Y down, media pixels, never mirrored | the `<video>` / `<img>` the session was given |
|
|
35
|
+
| **feed** | top-left, Y down, 0..1 | the camera texture; every landmark and SDK mask |
|
|
36
|
+
| **tracker** | center, X right, Y up, Z toward the viewer | hand, body, holistic nodes (`TrackerPoint`) |
|
|
37
|
+
| **viewport** | top-left, Y down, CSS pixels | the rendered canvas, pointer events via `clientToScreenPixels` |
|
|
38
|
+
| **screen** | center, Y up, 720×1280 design pixels | `ScreenSpaceUI`, `ScreenImage`, labels |
|
|
39
|
+
| **world** | three.js world; tracked content on z = 0 | `TrackingAnchor`, face mesh, iris quads |
|
|
40
|
+
|
|
41
|
+
The feed is the source **cover-cropped to the render size**, enlarged by
|
|
42
|
+
`perf.cameraZoom` (default 1.2) about the centre, and **mirrored for the
|
|
43
|
+
webcam only**. Image and video sources are drawn straight. Because the feed
|
|
44
|
+
fills the viewport, feed and viewport agree per axis; a landmark at feed
|
|
45
|
+
(0.5, 0.5) is the middle of the canvas on every aspect ratio.
|
|
46
|
+
|
|
47
|
+
## The feed crop, once
|
|
48
|
+
|
|
49
|
+
`FeedCrop` is the single description of source → feed. The session's blit
|
|
50
|
+
loop draws with it, `computeMaskUvTransform` fits source-space masks with it,
|
|
51
|
+
and the point conversions use it, so what the trackers saw and what you
|
|
52
|
+
compute agree.
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { computeFeedCrop, feedToSource, sourceToFeed } from "@vincentt-xr/sdk";
|
|
56
|
+
import { useXRFeedCrop } from "@vincentt-xr/sdk/low-level";
|
|
57
|
+
|
|
58
|
+
// Inside the provider: the live crop (null until media is bound).
|
|
59
|
+
const crop = useXRFeedCrop();
|
|
60
|
+
|
|
61
|
+
// Or build one yourself for an offline pipeline.
|
|
62
|
+
const offline = computeFeedCrop({
|
|
63
|
+
source: { width: 1280, height: 720 },
|
|
64
|
+
feed: { width: 720, height: 1280 },
|
|
65
|
+
zoom: 1.2,
|
|
66
|
+
mirrored: true, // webcam
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
// A landmark back onto the raw frame (for a crop, a server call, an overlay on the <video>).
|
|
70
|
+
const px = feedToSource({ x: 0.62, y: 0.4 }, crop); // { x, y } in source pixels
|
|
71
|
+
// A detection made on the raw frame into landmark space.
|
|
72
|
+
const fp = sourceToFeed({ x: 800, y: 300 }, crop); // { x, y } in 0..1
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`feedToSource` can return points outside the source when the feed shows
|
|
76
|
+
less than the frame (zoom > 1 never does; a contain-shaped source does);
|
|
77
|
+
`isFeedPointInSource` tells you.
|
|
78
|
+
|
|
79
|
+
## Mirroring
|
|
80
|
+
|
|
81
|
+
- Positions never need a flip: landmarks already live in the (possibly
|
|
82
|
+
mirrored) feed, and the feed is what you see.
|
|
83
|
+
- **Direction signs and side names** do. Every SDK left / right is the
|
|
84
|
+
person's own side on every source (GROUNDING "Left and right"); MediaPipe's
|
|
85
|
+
own sides (landmark indices, labels, blendshape names) are swapped on the
|
|
86
|
+
mirrored webcam. The SDK corrects them with `isMirroredSourceKind(sourceKind)`
|
|
87
|
+
/ `useXRMirrored()`; for your own reads use `personSide(side, mirrored)`,
|
|
88
|
+
`feedSidedName(name, mirrored)` and `personLeftSign(mirrored)` from
|
|
89
|
+
`@vincentt-xr/sdk`, never `sourceKind === "webcam"` inline.
|
|
90
|
+
- `mirrorFeedX` flips a feed point's x (for a direction vector, or to compare
|
|
91
|
+
a selfie feed with a straight recording).
|
|
92
|
+
|
|
93
|
+
## Camera orientation
|
|
94
|
+
|
|
95
|
+
A frame delivered sideways (a native capture, a rotated file) must be
|
|
96
|
+
rotated before it becomes the source. `orientSourcePoint(point, size,
|
|
97
|
+
orientation)` and `orientedSize(size, orientation)` move a point and a frame
|
|
98
|
+
size through a clockwise 0 / 90 / 180 / 270 rotation; apply them before
|
|
99
|
+
`sourceToFeed`. The browser already orients webcam frames, so the session
|
|
100
|
+
does not rotate.
|
|
101
|
+
|
|
102
|
+
## Conversions
|
|
103
|
+
|
|
104
|
+
| From → to | Helper |
|
|
105
|
+
| ---------------------- | ------------------------------------------------------------------- |
|
|
106
|
+
| source ↔ feed | `sourceToFeed`, `feedToSource` (with a `FeedCrop`) |
|
|
107
|
+
| feed ↔ tracker | `feedToTracker`, `trackerToFeed` |
|
|
108
|
+
| feed ↔ viewport | `feedToViewport`, `viewportToFeed` (viewport size) |
|
|
109
|
+
| feed ↔ screen | `feedToScreenPixels`, `screenPixelsToFeed` |
|
|
110
|
+
| feed / tracker → world | `feedToWorld`, `trackerToWorld` (camera, optional `planeZ`) |
|
|
111
|
+
| world → feed / tracker | `worldToFeed`, `worldToTracker` (`z` is NDC depth) |
|
|
112
|
+
| media layout | `computeFitLayout(source, destination, "cover" \| "contain", zoom)` |
|
|
113
|
+
|
|
114
|
+
The screen-space helpers (`normalizedToScreenPixels`, `worldToScreenPixels`,
|
|
115
|
+
`screenPointToRay`, `mediaToScreenPixels` …) are unchanged and documented in
|
|
116
|
+
[Screen-Space Coordinates](/api/screen-space-coordinates); a feed point is a
|
|
117
|
+
"normalized viewport point" there.
|
|
118
|
+
|
|
119
|
+
### Worked examples
|
|
120
|
+
|
|
121
|
+
**A hand joint to a DOM overlay**
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
const joint = handNode.hands?.[0].coordinates?.[8]; // tracker space
|
|
125
|
+
const feed = trackerToFeed(joint); // 0..1
|
|
126
|
+
const css = feedToViewport(feed, { width: el.clientWidth, height: el.clientHeight });
|
|
127
|
+
badge.style.transform = `translate(${css.x}px, ${css.y}px)`;
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
**A face landmark into the 3D scene** (what `TrackingAnchor` does)
|
|
131
|
+
|
|
132
|
+
```ts
|
|
133
|
+
useFrame(() => {
|
|
134
|
+
const tip = face.landmarks[1]; // feed space, from useFaceInfo
|
|
135
|
+
feedToWorld(tip, camera, mesh.position); // onto the z = 0 tracking plane
|
|
136
|
+
});
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
**A tap to the raw frame** (send the touched pixel to a server)
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
const screen = clientToScreenPixels({ point: { x: e.clientX, y: e.clientY }, rect });
|
|
143
|
+
const feed = screenPixelsToFeed(screen, viewportSize);
|
|
144
|
+
const sourcePx = feedToSource(feed, useXRFeedCrop()!);
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
**A mask made on the raw video**
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
computeMaskUvTransform({
|
|
151
|
+
space: "source",
|
|
152
|
+
maskWidth: 1280,
|
|
153
|
+
maskHeight: 720,
|
|
154
|
+
feedWidth,
|
|
155
|
+
feedHeight,
|
|
156
|
+
zoom: 1.2, // the session's perf.cameraZoom
|
|
157
|
+
mirrored: true, // webcam
|
|
158
|
+
});
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Regression coverage
|
|
162
|
+
|
|
163
|
+
`lib/coordinates/coordinates.test.ts` pins: the cover crop with zoom against
|
|
164
|
+
the blit loop's rectangle for landscape→portrait, portrait→landscape and 4:3
|
|
165
|
+
pairs; webcam mirroring (source left edge → feed right edge, round trip);
|
|
166
|
+
feed/tracker/viewport/screen round trips and agreement with
|
|
167
|
+
`trackerPointToScreenPixels`; feed→world depth independence and plane
|
|
168
|
+
selection; and the mask fit matching the feed crop, zoom included.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Cross-Camera & Orientation"
|
|
3
|
+
description: "Front / rear camera selection and switching, device enumeration, explicit mirror policy for preview and tracking, and orientation-safe alignment."
|
|
4
|
+
sidebarTitle: "Cross-Camera"
|
|
5
|
+
icon: "camera-rotate"
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Choosing and switching cameras
|
|
9
|
+
|
|
10
|
+
```tsx
|
|
11
|
+
import { XRMediaSource, listCameras, findCameraByFacing } from "@vincentt-xr/sdk";
|
|
12
|
+
|
|
13
|
+
// Bind: front (default) or rear, or a specific device.
|
|
14
|
+
await session.setMediaSource({ source: XRMediaSource.WEBCAM, facingMode: "environment" });
|
|
15
|
+
await session.setMediaSource({ source: XRMediaSource.WEBCAM, deviceId });
|
|
16
|
+
|
|
17
|
+
// Switch on a live session (models, scene and lifecycle stay; a bound or
|
|
18
|
+
// stopped session is started). Resolves to the facing the track reports.
|
|
19
|
+
const facing = await session.switchCamera({ facingMode: "environment" });
|
|
20
|
+
|
|
21
|
+
// Enumerate (labels appear once permission is granted).
|
|
22
|
+
const cameras = await listCameras(); // [{ deviceId, label, facing: "user" | "environment" | null }]
|
|
23
|
+
const rear = findCameraByFacing(cameras, "environment");
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`facingMode` is a soft hint: a laptop with one camera answers `environment`
|
|
27
|
+
with that camera, and `cameraFacing` then reports what the track says.
|
|
28
|
+
`deviceId` is exact and beats the hint. The same request again is a no-op;
|
|
29
|
+
a different facing, device or mirror policy releases the old stream and
|
|
30
|
+
tracks once and binds the new camera under the same source canvas, so every
|
|
31
|
+
model keeps its input binding. `useXRCameraFacing()` and
|
|
32
|
+
`useXRCameraDeviceId()` (low-level) read the result.
|
|
33
|
+
|
|
34
|
+
## Mirror policy
|
|
35
|
+
|
|
36
|
+
There is exactly one mirror decision per session and it is keyed on the
|
|
37
|
+
source kind:
|
|
38
|
+
|
|
39
|
+
| Source | `sourceKind` | Mirrored |
|
|
40
|
+
| ---------------------------- | ----------------- | -------- |
|
|
41
|
+
| front camera (`user`) | `webcam` | yes |
|
|
42
|
+
| rear camera (`environment`) | `rear-camera` | no |
|
|
43
|
+
| image / video stream, canvas | `image` / `video` | no |
|
|
44
|
+
|
|
45
|
+
The kind follows the _effective_ facing (what the track reports), so a
|
|
46
|
+
one-camera laptop that answers an `environment` request with its front
|
|
47
|
+
camera is still mirrored; `mirrored: true | false` on the bind pins it
|
|
48
|
+
either way. Preview and tracking read the same feed, so a landmark
|
|
49
|
+
unprojected to the scene lands where the preview shows it on every camera.
|
|
50
|
+
Consumers never flip positions; `useXRMirrored()` (low-level, the same
|
|
51
|
+
answer as `isMirroredSourceKind(sourceKind)`) is for _direction_ semantics
|
|
52
|
+
only, such as which hand is the person's left. Every tracker and the
|
|
53
|
+
coordinate toolkit already key off the kind, so every SDK left / right is the
|
|
54
|
+
person's own side on every camera (GROUNDING "Left and right").
|
|
55
|
+
|
|
56
|
+
## Orientation
|
|
57
|
+
|
|
58
|
+
`useXROrientation()` (low-level) reports `{ type: "portrait" | "landscape",
|
|
59
|
+
angle }` from `screen.orientation` (viewport aspect as the fallback). On a
|
|
60
|
+
change the session re-reads the live video size, so `useXRFeedCrop()` and
|
|
61
|
+
everything built on the coordinate toolkit re-align without waiting for the
|
|
62
|
+
next frame; the blit loop also resyncs the source size every frame. Content
|
|
63
|
+
anchored to landmarks (`TrackingAnchor`, `feedToWorld`, …) therefore stays
|
|
64
|
+
on the face through a rotation and a camera switch. Use
|
|
65
|
+
`calibration.orientation` only for sources the browser does not rotate
|
|
66
|
+
(a raw sideways capture).
|
|
67
|
+
|
|
68
|
+
## Lifecycle guarantees
|
|
69
|
+
|
|
70
|
+
- Switching releases the previous stream exactly once (`ended` from the
|
|
71
|
+
replaced tracks is ignored), keeps models initialized, and keeps
|
|
72
|
+
`running` / `paused` as it was; `switchCamera` additionally starts a
|
|
73
|
+
`media-bound` / `stopped` session.
|
|
74
|
+
- A switch that fails (device unplugged, in use) leaves the session where it
|
|
75
|
+
was with the error published; call again or pick another device.
|
|
76
|
+
- `session.dispose()` (provider unmount) removes the orientation listeners.
|
|
77
|
+
|
|
78
|
+
## Manual validation
|
|
79
|
+
|
|
80
|
+
Run `/camera` on a phone:
|
|
81
|
+
|
|
82
|
+
| Case | Steps | Expect |
|
|
83
|
+
| ------------------- | ----------------------------------- | ---------------------------------------------------------------------- |
|
|
84
|
+
| Front camera | load | `front · mirrored`; the cyan dot sits on the nose |
|
|
85
|
+
| Rear camera | "front → rear" | `rear · straight`; text in view reads correctly; dot still on the nose |
|
|
86
|
+
| Device list | after permission | labelled cameras; tapping one opens it (device id shown) |
|
|
87
|
+
| Mirror override | mirror = straight, "apply mirror" | front feed un-mirrored; dot still aligned |
|
|
88
|
+
| Rotate | turn the phone | orientation line flips, source size updates, dot stays aligned |
|
|
89
|
+
| Switch while paused | pause (session-health), switch here | stays paused; resume runs on the new camera |
|
|
@@ -32,11 +32,10 @@ from final preview/runtime surfaces.
|
|
|
32
32
|
|
|
33
33
|
## In-repo examples
|
|
34
34
|
|
|
35
|
-
- `/viewport-editor`: general 3D scene-object editor.
|
|
36
35
|
- `/unified-architecture`: unified face-tracking scene using
|
|
37
36
|
`XRSceneEditorProvider`, `XRSceneViewport`, hierarchy, inspector, toolbar, and
|
|
38
37
|
media switching.
|
|
39
|
-
- `/
|
|
38
|
+
- `/face-mesh?faces=two`: multi-face face mesh setup backed by the same reusable
|
|
40
39
|
editor primitives.
|
|
41
40
|
- `/debugger`: debugger overlay mounted beside a minimal tracked scene.
|
|
42
41
|
|
|
@@ -129,6 +128,28 @@ It renders sections for components present on the selected object:
|
|
|
129
128
|
|
|
130
129
|
All edits route through `onComponentChange` or `onObjectChange`.
|
|
131
130
|
|
|
131
|
+
## Docked windows
|
|
132
|
+
|
|
133
|
+
The hierarchy and inspector float by default: draggable glass cards over the
|
|
134
|
+
page. To put them in your own layout — a side panel or a settings drawer — pass
|
|
135
|
+
`docked`. A docked window renders in place: no portal, no fixed position, no
|
|
136
|
+
dragging. It takes the container's width, grows with its content, and drops the
|
|
137
|
+
glass card so it blends into the host.
|
|
138
|
+
|
|
139
|
+
```tsx
|
|
140
|
+
<XRSceneEditorProvider defaultStore={store}>
|
|
141
|
+
<aside className="my-side-panel">
|
|
142
|
+
<SceneHierarchyPanel docked />
|
|
143
|
+
<SceneInspectorPanel docked />
|
|
144
|
+
</aside>
|
|
145
|
+
<XRSceneViewport />
|
|
146
|
+
</XRSceneEditorProvider>
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
The provider must wrap the layout that hosts the docked panels. `docked` also
|
|
150
|
+
works on `HierarchyPanel`, `ObjectInspectorPanel`, `ScreenImageSettingsPanel`,
|
|
151
|
+
`ScreenTextSettingsPanel`, `ProductHotspotPanel` and `XRDiagnosticsPanel`.
|
|
152
|
+
|
|
132
153
|
## `ViewportToolbar`
|
|
133
154
|
|
|
134
155
|
Small optional toolbar for transient viewport controls.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Diagnostics & Calibration"
|
|
3
|
+
description: "Opt-in tracking diagnostics (readiness, inference time and rate, dropped frames, confidence, errors) and a device calibration workflow for orientation, mirror, scale, face distance and body proportions."
|
|
4
|
+
sidebarTitle: "Diagnostics & Calibration"
|
|
5
|
+
icon: "gauge-high"
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Tracking diagnostics
|
|
9
|
+
|
|
10
|
+
```tsx
|
|
11
|
+
import { useXRDiagnostics } from "@vincentt-xr/sdk/low-level";
|
|
12
|
+
import { XRDiagnosticsPanel } from "@vincentt-xr/sdk/debug-ui";
|
|
13
|
+
|
|
14
|
+
// Data (any component under <XRProvider>, outside the R3F canvas or inside):
|
|
15
|
+
const diagnostics = useXRDiagnostics({ intervalMs: 250, models: [XRModel.FACE_TRACKER] });
|
|
16
|
+
diagnostics.session.feedFps; // frames the session blits per second
|
|
17
|
+
diagnostics.models[0].state; // "unavailable" | "loading" | "ready" | "paused" | "error"
|
|
18
|
+
diagnostics.models[0].inferenceMs; // last inference, ms
|
|
19
|
+
diagnostics.models[0].inferenceFps; // inferences per second (~1 s window)
|
|
20
|
+
diagnostics.models[0].skippedFrames; // RAF ticks not analysed (throttle / busy)
|
|
21
|
+
diagnostics.models[0].detections; // faces / hands / bodies in the last inference
|
|
22
|
+
diagnostics.models[0].confidence; // primary-subject score, when the model has one
|
|
23
|
+
|
|
24
|
+
// Panel (debug UI, outside the canvas):
|
|
25
|
+
<XRDiagnosticsPanel models={[XRModel.FACE_TRACKER, XRModel.HAND_TRACKER]} />;
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The hook samples on an interval, never per frame, and only while a consumer
|
|
29
|
+
is mounted. Nothing is collected in production unless you render it, so it is
|
|
30
|
+
safe to ship behind a debug flag. The `models` list defaults to every
|
|
31
|
+
`XRModel`, so models nobody acquired show as `unavailable`.
|
|
32
|
+
|
|
33
|
+
Per-model fields come from the model instance (`getDiagnostics()` on every
|
|
34
|
+
`BaseXRModel`): inference timing is recorded around each MediaPipe call;
|
|
35
|
+
`detections` and `confidence` are what each model can report (face: count,
|
|
36
|
+
no score; hand / gesture: count and the handedness or gesture score; body /
|
|
37
|
+
holistic: count and mean landmark visibility; segmenters: 1 when a mask was
|
|
38
|
+
produced, no score). `targetFps` is the per-tracker cap when one was
|
|
39
|
+
requested. `owners` lists who holds the model and who wants it running.
|
|
40
|
+
|
|
41
|
+
Session fields mirror the store: lifecycle, ready, source and kind, camera
|
|
42
|
+
and feed sizes, feed fps and the session error. `inferenceMsTotal` sums the
|
|
43
|
+
last inference of every running model, a quick "how much ML per frame".
|
|
44
|
+
|
|
45
|
+
## Calibration
|
|
46
|
+
|
|
47
|
+
Cameras differ in orientation, mirroring and field of view, and people stand
|
|
48
|
+
at different distances. The calibration workflow measures a device once,
|
|
49
|
+
persists it, and applies it only where you ask. **Raw tracker results are
|
|
50
|
+
never altered.**
|
|
51
|
+
|
|
52
|
+
```tsx
|
|
53
|
+
import { useXRCalibration } from "@vincentt-xr/sdk/tracking";
|
|
54
|
+
|
|
55
|
+
const cal = useXRCalibration({ storageKey: "myapp.calibration" });
|
|
56
|
+
|
|
57
|
+
// Capture references while the person stands at their normal distance:
|
|
58
|
+
cal.calibrateFace(faceInfo.landmarks); // feed-space face landmarks
|
|
59
|
+
cal.calibrateBody(poseLandmarksInFeedSpace); // 33 pose landmarks, feed space
|
|
60
|
+
|
|
61
|
+
// Device transform:
|
|
62
|
+
cal.update({ orientation: 90, mirrored: "auto", scale: 1.1, offset: { x: 0, y: 0.02 } });
|
|
63
|
+
|
|
64
|
+
// Apply where it matters:
|
|
65
|
+
const nose = cal.applyToFeedPoint(faceInfo.landmarks[4]); // new point, input untouched
|
|
66
|
+
const distance = cal.faceDistance(faceInfo.landmarks); // 1 at reference, >1 closer
|
|
67
|
+
const bodyScale = cal.bodyScale(poseLandmarksInFeedSpace); // 1 at reference
|
|
68
|
+
|
|
69
|
+
cal.reset(); // defaults; clears the persisted value
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
| Field | Meaning |
|
|
73
|
+
| ----------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
|
74
|
+
| `orientation` | `0 / 90 / 180 / 270`: clockwise rotation applied to feed points. |
|
|
75
|
+
| `mirrored` | `"auto"` follows the session (only the webcam is mirrored); `true`/`false` flip only when they disagree with the session. |
|
|
76
|
+
| `scale`, `offset` | Uniform scale about the feed centre, then a feed-unit nudge. |
|
|
77
|
+
| `face` | `faceWidth` (temples 234↔454) and `eyeDistance` (33↔263) in feed-width units, plus `capturedAt`. |
|
|
78
|
+
| `body` | `shoulderWidth` (11↔12), `hipWidth` (23↔24), `torsoLength` (mid-shoulder to mid-hip). |
|
|
79
|
+
|
|
80
|
+
The pure helpers (`applyCalibrationToFeedPoint`, `measureFaceCalibration`,
|
|
81
|
+
`measureBodyCalibration`, `estimateFaceDistance`, `estimateBodyScale`,
|
|
82
|
+
`updateCalibration`, `resetCalibration`, `loadCalibration`,
|
|
83
|
+
`saveCalibration`, `clearCalibration`) work without React. Persistence uses
|
|
84
|
+
`localStorage` under `vincentt.xr.calibration` by default, validates every
|
|
85
|
+
field on load, and drops blobs from another `version`.
|
|
86
|
+
|
|
87
|
+
## Manual validation
|
|
88
|
+
|
|
89
|
+
Run `/session-health`:
|
|
90
|
+
|
|
91
|
+
| Case | Steps | Expect |
|
|
92
|
+
| ----------------- | ----------------------------- | ------------------------------------------------------------------------- |
|
|
93
|
+
| Diagnostics panel | load, pause / resume | face + body `ready`, inference ms / fps live; `paused` and 0 fps on pause |
|
|
94
|
+
| Unavailable model | untick "mount hand tracker" | hand row reads `unavailable`; tick again → `loading` → `ready` |
|
|
95
|
+
| Face calibration | "calibrate face", move closer | distance rises above 1; reload keeps the reference |
|
|
96
|
+
| Orientation | set 90° | calibrated nose point rotates; raw point unchanged |
|
|
97
|
+
| Reset | "reset" | defaults, distance `–`, storage cleared |
|