@vincentt-xr/sdk 2.0.0-alpha.0 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GROUNDING.md +1813 -27
- package/dist/{DebuggerUI-CEfdddkZ.cjs → DebuggerUI-B45_Gb9d.cjs} +1 -1
- package/dist/{DebuggerUI-Dgop6Q6o.js → DebuggerUI-Cq28c18s.js} +1 -1
- package/dist/FaceMeshRenderer-6t8OsQhK.js +1195 -0
- package/dist/FaceMeshRenderer-lQLhYIW-.cjs +155 -0
- package/dist/FaceRetouch-CDrL0pXg.js +551 -0
- package/dist/FaceRetouch-DtNl8-4e.cjs +134 -0
- package/dist/audio/AudioEngine.d.ts +3 -1
- package/dist/audio/AudioPlayer.d.ts +29 -0
- package/dist/audio/AudioPlayerLogic.d.ts +4 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/components/Axes/index.d.ts +6 -0
- package/dist/components/FaceMesh/FaceMeshRenderer.d.ts +17 -0
- package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +22 -1
- package/dist/components/FaceMesh/canonical/FaceMeshMaterial.d.ts +25 -0
- package/dist/components/FaceMesh/canonical/assets.d.ts +3 -0
- package/dist/components/FaceMesh/canonical/geometryGroups.d.ts +9 -0
- package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +2 -0
- package/dist/components/FaceMesh/canonical/shaders.d.ts +2 -0
- package/dist/components/FaceMesh/deform.d.ts +3 -0
- package/dist/components/FaceMesh/index.d.ts +5 -0
- package/dist/components/FaceMesh/material/defaults.d.ts +74 -0
- package/dist/components/FaceMesh/material/index.d.ts +1 -0
- package/dist/components/FaceMesh/material/options.d.ts +5 -1
- package/dist/components/FaceMesh/material/support.d.ts +7 -2
- package/dist/components/FaceMesh/material/textures.d.ts +4 -0
- package/dist/components/FaceMesh/material/types.d.ts +4 -2
- package/dist/components/FaceRetouch/Background.d.ts +9 -5
- package/dist/components/FaceRetouch/FaceRetouch.d.ts +47 -0
- package/dist/components/FaceRetouch/index.d.ts +2 -6
- package/dist/components/FaceRetouch/landmarkMask.d.ts +4 -6
- package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +4 -7
- package/dist/components/FaceTracker/index.d.ts +5 -3
- package/dist/components/FaceWarp3D/FaceWarp3DHead.d.ts +15 -0
- package/dist/components/FaceWarp3D/defaults.d.ts +37 -0
- package/dist/components/FaceWarp3D/index.d.ts +3 -0
- package/dist/components/FaceWarp3D/warp.d.ts +13 -0
- package/dist/components/HandJoints2D/index.d.ts +6 -0
- package/dist/components/HandTracker/index.d.ts +2 -2
- package/dist/components/HandTracking2D/HandVisual.d.ts +12 -0
- package/dist/components/HandTracking2D/index.d.ts +5 -0
- package/dist/components/HandTracking2DController/index.d.ts +5 -0
- package/dist/components/HandTracking3D/index.d.ts +6 -0
- package/dist/components/LightObject/index.d.ts +20 -0
- package/dist/components/LookAt/index.d.ts +23 -0
- package/dist/components/MeshRenderer/index.d.ts +58 -0
- package/dist/components/PerspectiveCamera/index.d.ts +2 -8
- package/dist/components/PinToMesh/index.d.ts +47 -0
- package/dist/components/RenderGroupLayer/index.d.ts +6 -0
- package/dist/components/SceneObjectRenderer/index.d.ts +16 -0
- package/dist/components/Screen2D/ScreenTransformFrame.d.ts +11 -0
- package/dist/components/Screen2D/ScreenTransformHelpers.d.ts +19 -0
- package/dist/components/Screen2D/ScreenTransformInteraction.d.ts +18 -0
- package/dist/components/Screen2D/index.d.ts +5 -0
- package/dist/components/ScreenGlass/defaults.d.ts +3 -0
- package/dist/components/ScreenGlass/index.d.ts +5 -0
- package/dist/components/ScreenGlass/shaders.d.ts +2 -0
- package/dist/components/ScreenGlass/types.d.ts +10 -0
- package/dist/components/ScreenGradient/defaults.d.ts +5 -0
- package/dist/components/ScreenGradient/index.d.ts +5 -0
- package/dist/components/ScreenGradient/shaders.d.ts +2 -0
- package/dist/components/ScreenGradient/types.d.ts +19 -0
- package/dist/components/ScreenImage/ScreenImage.d.ts +4 -0
- package/dist/components/ScreenImage/components/DynamicTextureSync.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageBody.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageFrame.d.ts +9 -0
- package/dist/components/ScreenImage/components/ScreenImageTransformFrame.d.ts +9 -0
- package/dist/components/ScreenImage/fit.d.ts +52 -0
- package/dist/components/ScreenImage/index.d.ts +8 -0
- package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts +9 -0
- package/dist/components/ScreenImage/materials/ScreenImageMatcap.d.ts +3 -0
- package/dist/components/ScreenImage/source.d.ts +16 -0
- package/dist/components/ScreenImage/types.d.ts +37 -0
- package/dist/components/ScreenImage/utils/ScreenImageConstants.d.ts +9 -0
- package/dist/components/ScreenImage/utils/ScreenImageHelpers.d.ts +4 -0
- package/dist/components/ScreenShape/geometry.d.ts +5 -0
- package/dist/components/ScreenShape/index.d.ts +5 -0
- package/dist/components/ScreenShape/types.d.ts +41 -0
- package/dist/components/ScreenSpaceUI/ScreenTransformGuide.d.ts +15 -0
- package/dist/components/ScreenSpaceUI/ScreenTransformOverlayGuide.d.ts +9 -0
- package/dist/components/ScreenSpaceUI/index.d.ts +2 -0
- package/dist/components/ScreenText/components/ScreenTextBody.d.ts +2 -0
- package/dist/components/ScreenText/components/ScreenTextTransformFrame.d.ts +9 -0
- package/dist/components/ScreenText/constants.d.ts +5 -0
- package/dist/components/ScreenText/index.d.ts +6 -0
- package/dist/components/ScreenText/types.d.ts +31 -0
- package/dist/components/ScreenText/utils/ScreenTextHelpers.d.ts +14 -0
- package/dist/components/ScreenText/utils/ScreenTextMappings.d.ts +17 -0
- package/dist/components/SequentialRenderPasses/index.d.ts +6 -0
- package/dist/components/TextLabel/TextLabel.d.ts +5 -0
- package/dist/components/TextLabel/alignment.d.ts +4 -0
- package/dist/components/TextLabel/bend.d.ts +9 -0
- package/dist/components/TextLabel/canvas/buildTextLabelTexture.d.ts +4 -0
- package/dist/components/TextLabel/canvas/canvasTypography.d.ts +13 -0
- package/dist/components/TextLabel/canvas/drawBackground.d.ts +21 -0
- package/dist/components/TextLabel/canvas/drawEffects.d.ts +34 -0
- package/dist/components/TextLabel/canvas/drawText.d.ts +19 -0
- package/dist/components/TextLabel/components/TextLabelMesh.d.ts +13 -0
- package/dist/components/TextLabel/constants.d.ts +2 -0
- package/dist/components/TextLabel/hooks/useTextLabelTexture.d.ts +7 -0
- package/dist/components/TextLabel/index.d.ts +2 -62
- package/dist/components/TextLabel/layout/measurement.d.ts +8 -0
- package/dist/components/TextLabel/layout/resolveTextLayout.d.ts +25 -0
- package/dist/components/TextLabel/layout/wrapText.d.ts +6 -0
- package/dist/components/TextLabel/types.d.ts +227 -0
- package/dist/components/TrackingAnchor/index.d.ts +18 -2
- package/dist/components/Transform3D/index.d.ts +30 -0
- package/dist/components/XRMediaSurface/index.d.ts +6 -6
- package/dist/components/XRScene/index.d.ts +14 -3
- package/dist/components/XRViewport/XRViewport.d.ts +11 -0
- package/dist/components/XRViewport/components/HiddenMedia.d.ts +7 -0
- package/dist/components/XRViewport/components/MediaAssetPicker.d.ts +8 -0
- package/dist/components/XRViewport/components/MediaControlStack.d.ts +13 -0
- package/dist/components/XRViewport/components/XRViewportLayers.d.ts +30 -0
- package/dist/components/XRViewport/constants.d.ts +24 -0
- package/dist/components/XRViewport/contexts.d.ts +6 -0
- package/dist/components/XRViewport/hooks/useViewportExternalFaces.d.ts +13 -0
- package/dist/components/XRViewport/hooks/useViewportMediaSource.d.ts +16 -0
- package/dist/components/XRViewport/hooks/useViewportMediaTexture.d.ts +10 -0
- package/dist/components/XRViewport/hooks/useViewportPipeline.d.ts +41 -0
- package/dist/components/XRViewport/hooks/useViewportRenderState.d.ts +47 -0
- package/dist/components/XRViewport/index.d.ts +3 -101
- package/dist/components/XRViewport/layout.d.ts +16 -0
- package/dist/components/XRViewport/slots.d.ts +16 -0
- package/dist/components/XRViewport/types.d.ts +89 -0
- package/dist/components/XRViewport/useViewportSize.d.ts +3 -0
- package/dist/components/index.d.ts +19 -0
- package/dist/contexts-BKvTIayr.cjs +1 -0
- package/dist/contexts-Bk5zz_VR.js +214 -0
- package/dist/debug-ui/component-editor/index.d.ts +19 -0
- package/dist/debug-ui/default-setting/index.d.ts +37 -0
- package/dist/debug-ui/default-setting.cjs +21 -0
- package/dist/debug-ui/default-setting.js +777 -0
- package/dist/debug-ui/face-mesh-renderer-settings/index.d.ts +8 -0
- package/dist/debug-ui/face-retouch/index.d.ts +11 -0
- package/dist/debug-ui/face-warp-3d/index.d.ts +6 -0
- package/dist/debug-ui/hand-joints-2d/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-2d/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-2d-controller/index.d.ts +6 -0
- package/dist/debug-ui/hand-tracking-3d/index.d.ts +6 -0
- package/dist/debug-ui/hierarchy-panel/index.d.ts +29 -0
- package/dist/debug-ui/index.d.ts +46 -0
- package/dist/debug-ui/layer-panel/index.d.ts +18 -0
- package/dist/debug-ui/media-source/MediaSourceSwitcher.d.ts +10 -0
- package/dist/debug-ui/media-source/index.d.ts +6 -0
- package/dist/debug-ui/media-source/media-preset.d.ts +9 -0
- package/dist/debug-ui/media-source/session.d.ts +12 -0
- package/dist/debug-ui/media-source.cjs +1 -0
- package/dist/debug-ui/media-source.js +9 -0
- package/dist/debug-ui/mesh-renderer-settings/index.d.ts +6 -0
- package/dist/debug-ui/object-inspector-panel/index.d.ts +25 -0
- package/dist/debug-ui/pin-to-mesh-settings/index.d.ts +10 -0
- package/dist/debug-ui/scene-editor/context.d.ts +38 -0
- package/dist/debug-ui/scene-editor/hierarchy-panel.d.ts +9 -0
- package/dist/debug-ui/scene-editor/index.d.ts +11 -0
- package/dist/debug-ui/scene-editor/inspector-panel.d.ts +7 -0
- package/dist/debug-ui/scene-editor/store-utils.d.ts +2 -0
- package/dist/debug-ui/scene-editor/viewport-toolbar.d.ts +4 -0
- package/dist/debug-ui/scene-editor/viewport.d.ts +21 -0
- package/dist/debug-ui/screen-glass-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-gradient-settings/GradientStopsEditor.d.ts +5 -0
- package/dist/debug-ui/screen-gradient-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-gradient-settings/styles.d.ts +3 -0
- package/dist/debug-ui/screen-image-settings/index.d.ts +9 -0
- package/dist/debug-ui/screen-shape-settings/index.d.ts +6 -0
- package/dist/debug-ui/screen-text-settings/NumberField.d.ts +9 -0
- package/dist/debug-ui/screen-text-settings/ScreenTextSettingsPanel.d.ts +3 -0
- package/dist/debug-ui/screen-text-settings/controls/NestedSection.d.ts +7 -0
- package/dist/debug-ui/screen-text-settings/controls/Section.d.ts +8 -0
- package/dist/debug-ui/screen-text-settings/controls/icons.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/index.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/BackgroundSection.d.ts +8 -0
- package/dist/debug-ui/screen-text-settings/sections/ContentSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/EffectsSection.d.ts +25 -0
- package/dist/debug-ui/screen-text-settings/sections/StyleSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/TransformSection.d.ts +5 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundFillControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundSizeControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/BackgroundStrokeControls.d.ts +2 -0
- package/dist/debug-ui/screen-text-settings/sections/background/types.d.ts +7 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/BendEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/EffectChrome.d.ts +21 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/GlowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/InnerShadowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/OutlineEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/ShadowEffectSection.d.ts +4 -0
- package/dist/debug-ui/screen-text-settings/sections/effects/types.d.ts +14 -0
- package/dist/debug-ui/screen-text-settings/settings.d.ts +90 -0
- package/dist/debug-ui/screen-text-settings/styles.d.ts +26 -0
- package/dist/debug-ui/screen-text-settings/transform.d.ts +6 -0
- package/dist/debug-ui/screen-text-settings/types.d.ts +9 -0
- package/dist/debug-ui/settings-panel/index.d.ts +38 -0
- package/dist/debug-ui/shared/MaterialSection.d.ts +13 -0
- package/dist/debug-ui/shared/ScreenTransformSection.d.ts +32 -0
- package/dist/debug-ui/transform-3d-settings/index.d.ts +11 -0
- package/dist/debug-ui.cjs +10 -0
- package/dist/debug-ui.js +8861 -0
- package/dist/drawNormalizedCrop-B-7zU-Xw.cjs +1 -0
- package/dist/drawNormalizedCrop-CRdIDyl1.js +44 -0
- package/dist/entries/all.d.ts +1 -0
- package/dist/entries/core.d.ts +55 -4
- package/dist/entries/face-effects.d.ts +12 -8
- package/dist/entries/internal.d.ts +1 -1
- package/dist/entries/low-level.d.ts +21 -4
- package/dist/entries/scene-object.d.ts +10 -0
- package/dist/entries/tracking.d.ts +25 -2
- package/dist/face-action/index.d.ts +34 -0
- package/dist/face-attribute-expression/index.d.ts +10 -0
- package/dist/face-effects.cjs +1 -252
- package/dist/face-effects.js +1404 -2613
- package/dist/face-info/blendshapes.d.ts +3 -0
- package/dist/face-info/derive.d.ts +8 -0
- package/dist/face-info/drawFaceCrop.d.ts +10 -0
- package/dist/face-info/getFaceInfo.d.ts +8 -0
- package/dist/face-info/index.d.ts +8 -0
- package/dist/face-info/landmarks.d.ts +21 -0
- package/dist/face-info/types.d.ts +52 -0
- package/dist/getFaceInfo-CyPAKh3h.cjs +1 -0
- package/dist/getFaceInfo-hUKMLFcr.js +118 -0
- package/dist/hand/handJoints2d.d.ts +49 -0
- package/dist/hand/handTracking2d.d.ts +33 -0
- package/dist/hand/handTracking2dController.d.ts +65 -0
- package/dist/hand/handTracking3d.d.ts +52 -0
- package/dist/hooks/faceResults.d.ts +1 -1
- package/dist/hooks/useCalibrationFaceTracker.d.ts +1 -1
- package/dist/hooks/useFaceAction.d.ts +3 -0
- package/dist/hooks/useFaceAttributeExpression.d.ts +16 -0
- package/dist/hooks/useFaceDetection.d.ts +18 -0
- package/dist/hooks/useFaceInfo.d.ts +7 -0
- package/dist/hooks/useFaceInfos.d.ts +15 -0
- package/dist/hooks/useFaceResultReader.d.ts +10 -0
- package/dist/hooks/useFrameCapture.d.ts +18 -0
- package/dist/hooks/useMediaRecorder.d.ts +14 -2
- package/dist/hooks/useOwnedTexture.d.ts +6 -0
- package/dist/hooks/useTrackerContext.d.ts +2 -2
- package/dist/hooks/useXRSession/facade.d.ts +3 -0
- package/dist/hooks/useXRSession/registry.d.ts +9 -0
- package/dist/index-82aENaRq.cjs +1 -0
- package/dist/{index-BD-JWMjE.js → index-BgQAX8rs.js} +198 -134
- package/dist/index-CEc1CAmB.cjs +1 -0
- package/dist/index-CTHeBQOs.js +178 -0
- package/dist/index-D4bLzBr6.cjs +1 -0
- package/dist/index-DSpzjuqd.cjs +1 -0
- package/dist/index-Di70KWuY.js +2049 -0
- package/dist/index-DzEzkBIH.cjs +17 -0
- package/dist/{index-CYMj26Ej.js → index-JygePOwo.js} +565 -498
- package/dist/index-UC0Ux38E.js +111 -0
- package/dist/{internal-KOKJDxCG.js → internal-DuMPpcbp.js} +86 -80
- package/dist/internal-i2JkM85N.cjs +1 -0
- package/dist/internal.cjs +1 -1
- package/dist/internal.js +2 -2
- package/dist/low-level.cjs +1 -1
- package/dist/low-level.js +959 -632
- package/dist/main.cjs +1 -1
- package/dist/main.js +556 -455
- package/dist/material/MaterialFill.d.ts +20 -0
- package/dist/material/MeshMaterial.d.ts +14 -0
- package/dist/material/index.d.ts +2 -0
- package/dist/models/Body/index.d.ts +1 -1
- package/dist/models/Face/index.d.ts +1 -0
- package/dist/models/Face/rotation.d.ts +3 -0
- package/dist/models/Hand/HandTracking.d.ts +2 -0
- package/dist/render/layerStack.d.ts +29 -0
- package/dist/render/renderGroups.d.ts +47 -0
- package/dist/render/renderPasses.d.ts +18 -0
- package/dist/renderGroups-C9QJrAXu.js +152 -0
- package/dist/renderGroups-Dvm1J4lW.cjs +1 -0
- package/dist/renderLayers-lDtL0_43.cjs +1 -0
- package/dist/renderLayers-zrK2pv5O.js +13 -0
- package/dist/renderPasses-CT5iBwtg.cjs +21 -0
- package/dist/renderPasses-CjgG_6kQ.js +1500 -0
- package/dist/scene-object/index.d.ts +146 -0
- package/dist/scene-object/runtimeRegistry.d.ts +14 -0
- package/dist/scene-object.cjs +1 -0
- package/dist/scene-object.js +40 -0
- package/dist/screen-image/screenImagePresets.d.ts +13 -0
- package/dist/screen-image/screenImageSettings.d.ts +57 -0
- package/dist/screen-text/screenTextPresets.d.ts +6 -0
- package/dist/screen-text/screenTextSettings.d.ts +147 -0
- package/dist/screen2d/coordinates.d.ts +87 -0
- package/dist/screen2d/index.d.ts +7 -0
- package/dist/screen2d/material-runtime.d.ts +32 -0
- package/dist/screen2d/screenTransform.d.ts +47 -0
- package/dist/screen2d/smoothing.d.ts +5 -0
- package/dist/screen2d/texture.d.ts +10 -0
- package/dist/screen2d/transform.d.ts +25 -0
- package/dist/screen2d/types.d.ts +90 -0
- package/dist/{selectors-Bub0o3qy.cjs → selectors-2QvO5RYY.cjs} +1 -1
- package/dist/{selectors-BHO6TnoC.js → selectors-Dw3C-gSk.js} +5 -5
- package/dist/services/MetricsCollector/index.d.ts +7 -1
- package/dist/services/MetricsCollector/utils.d.ts +1 -1
- package/dist/session-BChR4TgK.cjs +31 -0
- package/dist/session-C1StwN0f.js +363 -0
- package/dist/texture/animatedTexture.d.ts +61 -0
- package/dist/texture/animationMath.d.ts +8 -0
- package/dist/texture/sourceResolver.d.ts +30 -0
- package/dist/tracking.cjs +1 -1
- package/dist/tracking.js +582 -298
- package/dist/types/face.d.ts +1 -13
- package/dist/types/hand-tracking.d.ts +8 -0
- package/dist/types/index.d.ts +0 -1
- package/dist/types/nodes.d.ts +4 -1
- package/dist/types/session.d.ts +2 -0
- package/dist/types/tracking.d.ts +8 -2
- package/dist/types/util-types.d.ts +1 -1
- package/dist/useFaceInfo-C-WfMlYp.cjs +1 -0
- package/dist/useFaceInfo-CwsHZedC.js +62 -0
- package/dist/useFaceResults-CkFc1egZ.js +19 -0
- package/dist/useFaceResults-VcFdOUFV.cjs +1 -0
- package/dist/utils/drawNormalizedCrop.d.ts +22 -0
- package/dist/utils/integrity.d.ts +4 -25
- package/dist/utils/landmarkBounds.d.ts +1 -1
- package/dist/utils/logger.d.ts +2 -2
- package/dist/x-Co8jPhP0.js +127 -0
- package/dist/x-KC8E4Gaj.cjs +41 -0
- package/docs/MIGRATION_GUIDE.md +589 -0
- package/docs/SDK_DOCUMENTATION.md +1155 -0
- package/docs/api/audio.mdx +36 -0
- package/docs/api/behavior-components.mdx +164 -0
- package/docs/api/capture-recording.mdx +103 -0
- package/docs/api/debug-ui-scene-editor.mdx +249 -0
- package/docs/api/face-actions.mdx +104 -0
- package/docs/api/face-attribute-expression.mdx +87 -0
- package/docs/api/face-binding-anchors.mdx +53 -0
- package/docs/api/face-effects.mdx +19 -0
- package/docs/api/face-info.mdx +112 -0
- package/docs/api/face-mesh.mdx +98 -0
- package/docs/api/face-painting.mdx +33 -0
- package/docs/api/face-retouching.mdx +73 -0
- package/docs/api/material-render-state.mdx +132 -0
- package/docs/api/runtime-3d.mdx +311 -0
- package/docs/api/scene-object-store.mdx +213 -0
- package/docs/api/screen-image.mdx +41 -0
- package/docs/api/screen-shape.mdx +64 -0
- package/docs/api/screen-space-coordinates.mdx +41 -0
- package/docs/api/screen-text.mdx +39 -0
- package/docs/api/texture-sources.mdx +55 -0
- package/docs/api/trackers.mdx +46 -0
- package/docs/docs.json +70 -0
- package/docs/examples/scene-object-recipes.mdx +194 -0
- package/docs/guides/harness-debug-ui.mdx +139 -0
- package/docs/inspector/hierarchy-panel.mdx +51 -0
- package/docs/inspector/object-inspector-panel.mdx +51 -0
- package/docs/introduction.mdx +121 -0
- package/docs/scene-objects/overview.mdx +127 -0
- package/docs/scene-objects/render-groups.mdx +77 -0
- package/docs/scene-objects/runtime-rendering.mdx +75 -0
- package/docs/texture-api-consumer.md +58 -0
- package/package.json +55 -7
- package/dist/components/FaceRetouch/retouchRegions.d.ts +0 -23
- package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +0 -12
- package/dist/index-B5J4gzn2.cjs +0 -1
- package/dist/index-BVBVg96G.cjs +0 -1
- package/dist/index-DtwHElmG.js +0 -157
- package/dist/index-PtFmD6B8.cjs +0 -1
- package/dist/internal-DH5d1Xnq.cjs +0 -1
- package/dist/modules/XRClient/index.d.ts +0 -9
- package/dist/modules/index.d.ts +0 -1
- package/dist/services/ApiService.d.ts +0 -12
- package/dist/types/client.d.ts +0 -3
- package/dist/useFaceResults-cpsSmKKc.js +0 -81
- package/dist/useFaceResults-hVUJYSEY.cjs +0 -1
|
@@ -1,60 +1,52 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var h = (i, s, e) =>
|
|
4
|
-
import { jsx as
|
|
5
|
-
import {
|
|
6
|
-
import { FilesetResolver as
|
|
7
|
-
import * as
|
|
8
|
-
import { RGBAFormat as
|
|
9
|
-
var
|
|
10
|
-
|
|
11
|
-
/** Camera background and segmentation mask. `depthTest` disabled. */
|
|
12
|
-
BACKGROUND: -1e3,
|
|
13
|
-
/** Free world-space content. `depthTest` enabled. */
|
|
14
|
-
WORLD: 0,
|
|
15
|
-
/** Descendants of `FaceTracker` / `HandTracker` / `BodyTracker`. `depthTest` enabled. */
|
|
16
|
-
TRACKED: 100,
|
|
17
|
-
/** Descendants of `ScreenSpaceUI`. `depthTest` disabled. */
|
|
18
|
-
SCREEN_UI: 1e3
|
|
19
|
-
};
|
|
20
|
-
class F extends Error {
|
|
1
|
+
var Ae = Object.defineProperty;
|
|
2
|
+
var be = (i, s, e) => s in i ? Ae(i, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : i[s] = e;
|
|
3
|
+
var h = (i, s, e) => be(i, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { jsx as ne } from "react/jsx-runtime";
|
|
5
|
+
import { useRef as oe, useMemo as $, useEffect as Ee, useContext as ce, useCallback as ae, createElement as Ie, createContext as he } from "react";
|
|
6
|
+
import { FilesetResolver as V, PoseLandmarker as De, FaceLandmarker as _e, GestureRecognizer as Pe, HandLandmarker as Oe, ImageSegmenter as Ue, DrawingUtils as ze } from "@mediapipe/tasks-vision";
|
|
7
|
+
import * as E from "three";
|
|
8
|
+
import { RGBAFormat as de, LinearFilter as q, CanvasTexture as ue, SRGBColorSpace as Le } from "three";
|
|
9
|
+
var F = /* @__PURE__ */ ((i) => (i.WEBCAM = "webcam", i.STREAM = "stream", i.CANVAS = "canvas", i))(F || {}), z = /* @__PURE__ */ ((i) => (i.HAND_TRACKER = "vincentt.xr.hand-tracker", i.GESTURE_TRACKER = "vincentt.xr.gesture-tracker", i.FACE_TRACKER = "vincentt.xr.face-tracker", i.BODY_TRACKER = "vincentt.xr.body-tracker", i.BACKGROUND_SEGMENTATION = "vincentt.xr.background-segmentation", i))(z || {});
|
|
10
|
+
class b extends Error {
|
|
21
11
|
constructor(s) {
|
|
22
12
|
super(s), this.name = "VincenttXRError";
|
|
23
13
|
}
|
|
24
14
|
}
|
|
25
|
-
const
|
|
15
|
+
const Be = (i, s) => s, I = {
|
|
16
|
+
apiBaseUrl: Be(void 0, "https://api.vincentt.studio"),
|
|
26
17
|
// Pinned to the CDN, NOT build-env-driven: a stale VITE_ASSET_STORAGE_ENDPOINT
|
|
27
18
|
// baked a now-deleted CloudFront domain into every published bundle, breaking
|
|
28
19
|
// WASM/model loads at runtime. The host must never come from a mutable env value.
|
|
29
20
|
assetStorageEndpoint: "https://cdn.vincentt.studio",
|
|
30
21
|
// Versioned bucket for SDK-shipped reference assets (canonical face model, face textures, etc.).
|
|
31
22
|
sdkAssetsBaseUrl: "https://cdn.vincentt.studio/assets/sdk/face/v1",
|
|
32
|
-
loggerPrefix: "[vincentt-xr]:"
|
|
23
|
+
loggerPrefix: "[vincentt-xr]:",
|
|
24
|
+
sessionCheckInterval: 3e4
|
|
33
25
|
};
|
|
34
|
-
class
|
|
26
|
+
class Ve {
|
|
35
27
|
constructor(s) {
|
|
36
28
|
h(this, "options");
|
|
37
29
|
const e = {
|
|
38
30
|
verbose: !1,
|
|
39
|
-
prefix:
|
|
31
|
+
prefix: I.loggerPrefix
|
|
40
32
|
};
|
|
41
33
|
this.options = { ...e, ...s };
|
|
42
34
|
}
|
|
43
35
|
log(...s) {
|
|
44
36
|
const { verbose: e, prefix: t } = this.options;
|
|
45
|
-
e && console.
|
|
37
|
+
e && console.info(t, ...s);
|
|
46
38
|
}
|
|
47
39
|
error(...s) {
|
|
48
40
|
const { verbose: e, prefix: t } = this.options;
|
|
49
41
|
e && console.error(t, ...s);
|
|
50
42
|
}
|
|
51
43
|
}
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
const r = -t.position.z /
|
|
55
|
-
return
|
|
56
|
-
},
|
|
57
|
-
class
|
|
44
|
+
const O = (i) => i * 2 - 1, j = new E.Vector3(), Y = new E.Vector3(), H = ({ x: i, y: s, z: e }, t, a) => {
|
|
45
|
+
j.set((i - 0.5) * 2, (0.5 - s) * 2, -e), j.unproject(t), Y.copy(j).sub(t.position).normalize();
|
|
46
|
+
const r = -t.position.z / Y.z;
|
|
47
|
+
return a.copy(t.position).addScaledVector(Y, r), a;
|
|
48
|
+
}, It = (i, s) => H(i, s, new E.Vector3()), Ke = (i, s, e = 0.2) => e * s + (1 - e) * i;
|
|
49
|
+
class Z {
|
|
58
50
|
constructor(s) {
|
|
59
51
|
h(this, "canvas");
|
|
60
52
|
this.maxDimension = s;
|
|
@@ -65,52 +57,71 @@ class G {
|
|
|
65
57
|
* 2D context can't be obtained). The returned canvas is reused across calls.
|
|
66
58
|
*/
|
|
67
59
|
draw(s) {
|
|
68
|
-
const e = s.width, t = s.height,
|
|
69
|
-
if (
|
|
70
|
-
const r = Math.max(1, Math.round(e *
|
|
71
|
-
(!this.canvas || this.canvas.width !== r || this.canvas.height !==
|
|
72
|
-
const
|
|
73
|
-
return
|
|
60
|
+
const e = s.width, t = s.height, a = Math.min(1, this.maxDimension / Math.max(e, t));
|
|
61
|
+
if (a >= 1) return s;
|
|
62
|
+
const r = Math.max(1, Math.round(e * a)), o = Math.max(1, Math.round(t * a));
|
|
63
|
+
(!this.canvas || this.canvas.width !== r || this.canvas.height !== o) && (this.canvas = new OffscreenCanvas(r, o));
|
|
64
|
+
const u = this.canvas.getContext("2d");
|
|
65
|
+
return u ? (u.setTransform(1, 0, 0, 1, 0, 0), u.clearRect(0, 0, r, o), u.drawImage(s, 0, 0, r, o), this.canvas) : s;
|
|
74
66
|
}
|
|
75
67
|
dispose() {
|
|
76
68
|
this.canvas = void 0;
|
|
77
69
|
}
|
|
78
70
|
}
|
|
79
|
-
const
|
|
80
|
-
|
|
71
|
+
const Ne = {
|
|
72
|
+
"/assets/xr/models/face_tracking.task": "sha384-tYQh+yJE8llY+zO4RviZmWaSKs5W9tsB0ix5rjQVpF5/CLTqmwtP7bYtkUi2w65P",
|
|
73
|
+
"/assets/xr/models/hand_tracking.task": "sha384-uWvruVKd887ov8k43S+DBHCsi7UXXS+CKvvdM1PX00rjPzx/B0QrAPfJ1U9yKcze",
|
|
74
|
+
"/assets/xr/models/gesture_tracking.task": "sha384-2zuseP9JKKWAYlWQGKA2N+ooabB8V/yIxu23Mv8mtY7SowcySB+xKiOmqBhVvlUC",
|
|
75
|
+
"/assets/xr/models/body_tracking.task": "sha384-3Wk9dvRYo36b3Nc9xVjKamb4wCKt9FwiPjcHKfrvyJL/jXoGs+uRLnii6yjWLKul",
|
|
76
|
+
"/assets/xr/models/segmentation.tflite": "sha384-A2bQ8+iZyQM3KIGVBUf0FyetqxuynyKpjsjCE8uzoJEnj1ghCu6ge2daiRffAiS5",
|
|
77
|
+
"/assets/xr/3d-models/head-occluder-v2.glb": "sha384-+0G60Q6EBVE6hpITxGzQgofAeKexPltpsfPq5T5et9GGBjRhFP5HOdsKAhe73OaI",
|
|
78
|
+
"/assets/sdk/face/v1/canonical_face_model.obj": "sha384-xP8YM/72TXKvbMptXB9a8JUlUf3MZQ8uLurExrsZc05v6/P5RrsaNGMUYS/BZx8A",
|
|
79
|
+
"/opencv/facial_expression_recognition/resolve/main/facial_expression_recognition_mobilefacenet_2022july_int8.onnx": "sha384-g8mtFubiIc/yhqi591RsxD76AJLw2CfYgKhKbVkIiFlwBmIyG7ZAIVyGPXafrDGB"
|
|
80
|
+
};
|
|
81
|
+
async function He(i) {
|
|
81
82
|
const s = await fetch(i);
|
|
82
83
|
if (!s.ok)
|
|
83
84
|
throw new Error(`fetchAndVerify: ${s.status} ${i}`);
|
|
84
|
-
const e = await s.arrayBuffer(), t =
|
|
85
|
-
if (!t)
|
|
85
|
+
const e = await s.arrayBuffer(), t = Ge(i);
|
|
86
|
+
if (!t) {
|
|
87
|
+
if (qe(i))
|
|
88
|
+
throw new Error(`[xr-sdk integrity] no SRI hash for SDK asset ${i}`);
|
|
86
89
|
return console.warn(`[xr-sdk integrity] no SRI hash for ${i} — passing through`), e;
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
}
|
|
91
|
+
const a = await We(e);
|
|
92
|
+
if (a !== t)
|
|
89
93
|
throw new Error(
|
|
90
|
-
`[xr-sdk integrity] SRI mismatch for ${i}: expected ${t}, got ${
|
|
94
|
+
`[xr-sdk integrity] SRI mismatch for ${i}: expected ${t}, got ${a}`
|
|
91
95
|
);
|
|
92
96
|
return e;
|
|
93
97
|
}
|
|
94
|
-
|
|
95
|
-
|
|
98
|
+
function qe(i) {
|
|
99
|
+
try {
|
|
100
|
+
return new URL(i).origin === new URL(I.assetStorageEndpoint).origin;
|
|
101
|
+
} catch {
|
|
102
|
+
return !1;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
async function K(i) {
|
|
106
|
+
const s = await He(i);
|
|
96
107
|
return new Uint8Array(s);
|
|
97
108
|
}
|
|
98
|
-
function
|
|
109
|
+
function Ge(i) {
|
|
99
110
|
try {
|
|
100
111
|
const s = new URL(i);
|
|
101
|
-
return
|
|
112
|
+
return Ne[s.pathname];
|
|
102
113
|
} catch {
|
|
103
114
|
return;
|
|
104
115
|
}
|
|
105
116
|
}
|
|
106
|
-
async function
|
|
117
|
+
async function We(i) {
|
|
107
118
|
const s = await crypto.subtle.digest("SHA-384", i), e = new Uint8Array(s);
|
|
108
119
|
let t = "";
|
|
109
|
-
for (let
|
|
110
|
-
t += String.fromCharCode(e[
|
|
120
|
+
for (let a = 0; a < e.byteLength; a++)
|
|
121
|
+
t += String.fromCharCode(e[a]);
|
|
111
122
|
return `sha384-${btoa(t)}`;
|
|
112
123
|
}
|
|
113
|
-
class
|
|
124
|
+
class N {
|
|
114
125
|
constructor() {
|
|
115
126
|
h(this, "disposed", !1);
|
|
116
127
|
h(this, "paused", !1);
|
|
@@ -172,37 +183,40 @@ class L {
|
|
|
172
183
|
this.disposed = !0, this.rafId !== null && (cancelAnimationFrame(this.rafId), this.rafId = null);
|
|
173
184
|
}
|
|
174
185
|
}
|
|
175
|
-
const
|
|
176
|
-
class
|
|
186
|
+
const Xe = 768;
|
|
187
|
+
class je extends N {
|
|
177
188
|
constructor() {
|
|
178
189
|
super();
|
|
179
190
|
h(this, "landmarker", null);
|
|
180
191
|
h(this, "processingTime", null);
|
|
181
192
|
h(this, "coordinates");
|
|
182
193
|
h(this, "sourceCanvas");
|
|
183
|
-
h(this, "downscaler", new
|
|
194
|
+
h(this, "downscaler", new Z(Xe));
|
|
184
195
|
}
|
|
185
196
|
async getBodyData() {
|
|
186
197
|
if (!this.landmarker || !this.sourceCanvas)
|
|
187
198
|
return;
|
|
188
|
-
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(),
|
|
199
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), a = this.landmarker.detectForVideo(e, t), r = performance.now();
|
|
189
200
|
this.processingTime = r - t;
|
|
190
|
-
const
|
|
191
|
-
if (
|
|
192
|
-
return
|
|
193
|
-
d
|
|
194
|
-
const
|
|
195
|
-
d
|
|
201
|
+
const o = a.landmarks, u = [];
|
|
202
|
+
if (o)
|
|
203
|
+
return o.forEach((m, d) => {
|
|
204
|
+
u[d] || (u[d] = []), m.forEach((S) => {
|
|
205
|
+
const R = O(S.x), M = O(1 - S.y), x = O(S.z);
|
|
206
|
+
u[d].push({ x: R, y: M, z: x });
|
|
196
207
|
});
|
|
197
208
|
}), {
|
|
198
|
-
coordinatesList:
|
|
209
|
+
coordinatesList: u
|
|
199
210
|
};
|
|
200
211
|
}
|
|
201
212
|
async processBodyTracking() {
|
|
202
|
-
if (this.disposed || !this.landmarker)
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
213
|
+
if (!(this.disposed || this.paused || !this.landmarker)) {
|
|
214
|
+
if (this.shouldRunInference()) {
|
|
215
|
+
const e = await this.getBodyData();
|
|
216
|
+
e && (this.coordinates = e.coordinatesList);
|
|
217
|
+
}
|
|
218
|
+
this.disposed || this.paused || (this.rafId = requestAnimationFrame(this.processBodyTracking.bind(this)));
|
|
219
|
+
}
|
|
206
220
|
}
|
|
207
221
|
resume() {
|
|
208
222
|
this.paused && (super.resume(), this.processBodyTracking());
|
|
@@ -222,22 +236,22 @@ class Oe extends L {
|
|
|
222
236
|
return this.coordinates[e];
|
|
223
237
|
}
|
|
224
238
|
async initialize(e) {
|
|
225
|
-
const t = await
|
|
226
|
-
new URL("/assets/xr/wasm",
|
|
227
|
-
),
|
|
228
|
-
new URL("/assets/xr/models/body_tracking.task",
|
|
239
|
+
const t = await V.forVisionTasks(
|
|
240
|
+
new URL("/assets/xr/wasm", I.assetStorageEndpoint).toString()
|
|
241
|
+
), a = await K(
|
|
242
|
+
new URL("/assets/xr/models/body_tracking.task", I.assetStorageEndpoint).toString()
|
|
229
243
|
);
|
|
230
|
-
this.landmarker = await
|
|
244
|
+
this.landmarker = await De.createFromOptions(t, {
|
|
231
245
|
baseOptions: {
|
|
232
|
-
modelAssetBuffer:
|
|
246
|
+
modelAssetBuffer: a,
|
|
233
247
|
delegate: "GPU"
|
|
234
248
|
},
|
|
235
249
|
runningMode: "VIDEO",
|
|
236
250
|
numPoses: (e == null ? void 0 : e.numPoses) || 1
|
|
237
251
|
});
|
|
238
252
|
}
|
|
239
|
-
async run({ sourceCanvas: e }) {
|
|
240
|
-
this.sourceCanvas = e, this.rafId === null && await this.processBodyTracking();
|
|
253
|
+
async run({ sourceCanvas: e, targetFps: t }) {
|
|
254
|
+
this.sourceCanvas = e, this.setTargetFps(t), this.rafId === null && await this.processBodyTracking();
|
|
241
255
|
}
|
|
242
256
|
getModelNode() {
|
|
243
257
|
return {
|
|
@@ -251,8 +265,8 @@ class Oe extends L {
|
|
|
251
265
|
};
|
|
252
266
|
}
|
|
253
267
|
}
|
|
254
|
-
const
|
|
255
|
-
class
|
|
268
|
+
const Ye = 0.11, Qe = 4e-3, $e = (i, s) => s.set(i.x, -i.y, i.z);
|
|
269
|
+
class Ze extends N {
|
|
256
270
|
constructor() {
|
|
257
271
|
super();
|
|
258
272
|
h(this, "landmarker", null);
|
|
@@ -277,9 +291,9 @@ class Le extends L {
|
|
|
277
291
|
h(this, "landmarkScratch", { x: 0, y: 0, z: 0, visibility: 0 });
|
|
278
292
|
// Reused per-frame rotation scratch (matrix -> quaternion -> mirror -> euler),
|
|
279
293
|
// allocated once to avoid per-frame churn at 30+ fps.
|
|
280
|
-
h(this, "rotMatrixScratch", new
|
|
281
|
-
h(this, "rotQuatScratch", new
|
|
282
|
-
h(this, "rotEulerScratch", new
|
|
294
|
+
h(this, "rotMatrixScratch", new E.Matrix4());
|
|
295
|
+
h(this, "rotQuatScratch", new E.Quaternion());
|
|
296
|
+
h(this, "rotEulerScratch", new E.Euler());
|
|
283
297
|
}
|
|
284
298
|
// For non-webcam sources MediaPipe landmarks read as mirrored; flip x in
|
|
285
299
|
// place into the reused scratch so downstream unprojection sees the
|
|
@@ -289,7 +303,7 @@ class Le extends L {
|
|
|
289
303
|
return t ? (this.landmarkScratch.x = 1 - e.x, this.landmarkScratch.y = e.y, this.landmarkScratch.z = e.z, this.landmarkScratch.visibility = e.visibility, this.landmarkScratch) : e;
|
|
290
304
|
}
|
|
291
305
|
extractAndUpdateFaceData(e) {
|
|
292
|
-
var
|
|
306
|
+
var a;
|
|
293
307
|
if (!e.faceLandmarks.length) {
|
|
294
308
|
this.faces = [];
|
|
295
309
|
return;
|
|
@@ -298,46 +312,42 @@ class Le extends L {
|
|
|
298
312
|
for (this.faces.length > t && (this.faces.length = t, this.leftCheekScratch.length = t, this.rightCheekScratch.length = t); this.faces.length < t; )
|
|
299
313
|
this.faces.push({
|
|
300
314
|
transform: {
|
|
301
|
-
position: new
|
|
302
|
-
rotation: new
|
|
303
|
-
scale: new
|
|
315
|
+
position: new E.Vector3(),
|
|
316
|
+
rotation: new E.Vector3(),
|
|
317
|
+
scale: new E.Vector3()
|
|
304
318
|
},
|
|
305
319
|
gestures: {}
|
|
306
|
-
}), this.leftCheekScratch.push(new
|
|
320
|
+
}), this.leftCheekScratch.push(new E.Vector3()), this.rightCheekScratch.push(new E.Vector3());
|
|
307
321
|
for (let r = 0; r < t; r += 1) {
|
|
308
322
|
if (!this.cameras[r])
|
|
309
323
|
continue;
|
|
310
|
-
const
|
|
311
|
-
this.rotMatrixScratch.fromArray(
|
|
324
|
+
const o = this.faces[r], u = (a = this.getSource) == null ? void 0 : a.call(this), m = !(u === "webcam" || !u), d = e.faceLandmarks[r], S = e.facialTransformationMatrixes[r];
|
|
325
|
+
this.rotMatrixScratch.fromArray(S.data), this.rotQuatScratch.setFromRotationMatrix(this.rotMatrixScratch), this.rotQuatScratch.set(
|
|
312
326
|
-this.rotQuatScratch.x,
|
|
313
327
|
this.rotQuatScratch.y,
|
|
314
328
|
this.rotQuatScratch.z,
|
|
315
329
|
this.rotQuatScratch.w
|
|
316
|
-
), this.rotEulerScratch.setFromQuaternion(this.rotQuatScratch, "XYZ"),
|
|
317
|
-
|
|
318
|
-
this.rotEulerScratch.y,
|
|
319
|
-
this.rotEulerScratch.z
|
|
320
|
-
), V(
|
|
321
|
-
this.landmarkForSource(u[4], m),
|
|
330
|
+
), this.rotEulerScratch.setFromQuaternion(this.rotQuatScratch, "XYZ"), $e(this.rotEulerScratch, o.transform.rotation), H(
|
|
331
|
+
this.landmarkForSource(d[4], m),
|
|
322
332
|
this.cameras[r],
|
|
323
|
-
|
|
324
|
-
),
|
|
325
|
-
this.landmarkForSource(
|
|
333
|
+
o.transform.position
|
|
334
|
+
), H(
|
|
335
|
+
this.landmarkForSource(d[50], m),
|
|
326
336
|
this.cameras[r],
|
|
327
337
|
this.leftCheekScratch[r]
|
|
328
|
-
),
|
|
329
|
-
this.landmarkForSource(
|
|
338
|
+
), H(
|
|
339
|
+
this.landmarkForSource(d[280], m),
|
|
330
340
|
this.cameras[r],
|
|
331
341
|
this.rightCheekScratch[r]
|
|
332
342
|
);
|
|
333
|
-
const
|
|
334
|
-
isNaN(
|
|
335
|
-
|
|
343
|
+
const M = this.leftCheekScratch[r].distanceTo(this.rightCheekScratch[r]) * Ye, x = Math.abs(E.MathUtils.radToDeg(this.rotEulerScratch.y)), g = M + x * Qe, f = o.transform.scale.x, w = Ke(
|
|
344
|
+
isNaN(f) ? 0 : f,
|
|
345
|
+
g
|
|
336
346
|
);
|
|
337
|
-
|
|
338
|
-
const
|
|
339
|
-
for (const
|
|
340
|
-
|
|
347
|
+
o.transform.scale.set(w, w, w);
|
|
348
|
+
const p = o.gestures, k = e.faceBlendshapes[r].categories;
|
|
349
|
+
for (const C of k)
|
|
350
|
+
p[C.categoryName] = C.score;
|
|
341
351
|
}
|
|
342
352
|
}
|
|
343
353
|
async processFaceTracking() {
|
|
@@ -347,18 +357,30 @@ class Le extends L {
|
|
|
347
357
|
this.rafId = requestAnimationFrame(this.processFaceTracking.bind(this));
|
|
348
358
|
return;
|
|
349
359
|
}
|
|
350
|
-
const e =
|
|
351
|
-
(!this.analysisCanvas || this.analysisCanvas.width !==
|
|
360
|
+
const e = Je(this.numFaces), t = this.sourceCanvas.width, a = this.sourceCanvas.height, r = Math.min(1, e / Math.max(t, a)), o = Math.max(1, Math.round(t * r)), u = Math.max(1, Math.round(a * r));
|
|
361
|
+
(!this.analysisCanvas || this.analysisCanvas.width !== o || this.analysisCanvas.height !== u) && (this.analysisCanvas = new OffscreenCanvas(o, u));
|
|
352
362
|
const m = this.analysisCanvas.getContext("2d");
|
|
353
|
-
if (!m)
|
|
363
|
+
if (!m) {
|
|
364
|
+
this.rafId = requestAnimationFrame(this.processFaceTracking.bind(this));
|
|
354
365
|
return;
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
366
|
+
}
|
|
367
|
+
m.setTransform(1, 0, 0, 1, 0, 0), m.clearRect(0, 0, o, u), m.drawImage(this.sourceCanvas, 0, 0, o, u);
|
|
368
|
+
const d = performance.now(), S = this.landmarker.detectForVideo(this.analysisCanvas, d), R = performance.now();
|
|
369
|
+
this.processingTime = R - d, this.detectionResults = S, this.extractAndUpdateFaceData(S), !(this.disposed || this.paused) && (this.rafId = requestAnimationFrame(this.processFaceTracking.bind(this)));
|
|
358
370
|
}
|
|
359
371
|
resume() {
|
|
360
372
|
this.paused && (super.resume(), this.processFaceTracking());
|
|
361
373
|
}
|
|
374
|
+
dispose() {
|
|
375
|
+
if (super.dispose(), this.landmarker) {
|
|
376
|
+
try {
|
|
377
|
+
this.landmarker.close();
|
|
378
|
+
} catch {
|
|
379
|
+
}
|
|
380
|
+
this.landmarker = null;
|
|
381
|
+
}
|
|
382
|
+
this.sourceCanvas = void 0, this.analysisCanvas = void 0, this.getSource = void 0, this.detectionResults = void 0, this.processingTime = null, this.cameras = [], this.faces = [], this.leftCheekScratch = [], this.rightCheekScratch = [];
|
|
383
|
+
}
|
|
362
384
|
getFace(e = 0) {
|
|
363
385
|
return this.faces[e];
|
|
364
386
|
}
|
|
@@ -366,23 +388,23 @@ class Le extends L {
|
|
|
366
388
|
return this.detectionResults;
|
|
367
389
|
}
|
|
368
390
|
getRawFaceTrackingData(e = 0) {
|
|
369
|
-
var t,
|
|
391
|
+
var t, a, r, o;
|
|
370
392
|
if (this.detectionResults)
|
|
371
393
|
return {
|
|
372
394
|
landmarks: (t = this.detectionResults.faceLandmarks) == null ? void 0 : t[e],
|
|
373
|
-
blendshapes: (r = (
|
|
374
|
-
transformationMatrix: (
|
|
395
|
+
blendshapes: (r = (a = this.detectionResults.faceBlendshapes) == null ? void 0 : a[e]) == null ? void 0 : r.categories,
|
|
396
|
+
transformationMatrix: (o = this.detectionResults.facialTransformationMatrixes) == null ? void 0 : o[e]
|
|
375
397
|
};
|
|
376
398
|
}
|
|
377
399
|
async initialize(e) {
|
|
378
400
|
this.numFaces = (e == null ? void 0 : e.numFaces) || 1;
|
|
379
|
-
const t = await
|
|
380
|
-
new URL("/assets/xr/wasm",
|
|
381
|
-
),
|
|
382
|
-
new URL("/assets/xr/models/face_tracking.task",
|
|
383
|
-
), r = await
|
|
401
|
+
const t = await V.forVisionTasks(
|
|
402
|
+
new URL("/assets/xr/wasm", I.assetStorageEndpoint).toString()
|
|
403
|
+
), a = await K(
|
|
404
|
+
new URL("/assets/xr/models/face_tracking.task", I.assetStorageEndpoint).toString()
|
|
405
|
+
), r = await _e.createFromOptions(t, {
|
|
384
406
|
baseOptions: {
|
|
385
|
-
modelAssetBuffer:
|
|
407
|
+
modelAssetBuffer: a,
|
|
386
408
|
delegate: "GPU"
|
|
387
409
|
},
|
|
388
410
|
// Keep detection permissive for calibration workflows with dark webcam input.
|
|
@@ -394,25 +416,28 @@ class Le extends L {
|
|
|
394
416
|
runningMode: "VIDEO",
|
|
395
417
|
numFaces: this.numFaces
|
|
396
418
|
});
|
|
419
|
+
if (this.disposed) {
|
|
420
|
+
r.close();
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
397
423
|
this.landmarker = r;
|
|
398
424
|
}
|
|
399
|
-
async run({ sourceCanvas: e, getSource: t, targetFps:
|
|
400
|
-
this.sourceCanvas = e, this.getSource = t, this.setTargetFps(
|
|
425
|
+
async run({ sourceCanvas: e, getSource: t, targetFps: a }) {
|
|
426
|
+
this.sourceCanvas = e, this.getSource = t, this.setTargetFps(a), this.analysisCanvas = void 0, this.rafId === null && this.processFaceTracking();
|
|
401
427
|
}
|
|
402
428
|
getModelNode() {
|
|
403
|
-
const e = this
|
|
429
|
+
const e = this;
|
|
430
|
+
return {
|
|
404
431
|
get detectionResults() {
|
|
405
432
|
return e.detectionResults;
|
|
406
433
|
},
|
|
407
434
|
getDetectionResults: this.getDetectionResults.bind(this),
|
|
408
435
|
getFace: this.getFace.bind(this),
|
|
409
|
-
getRawFaceTrackingData: this.getRawFaceTrackingData.bind(this)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
_setCamera: (n, r) => {
|
|
413
|
-
this.cameras[r] = n;
|
|
436
|
+
getRawFaceTrackingData: this.getRawFaceTrackingData.bind(this),
|
|
437
|
+
setCamera: (a, r = 0) => {
|
|
438
|
+
this.cameras[r] = a;
|
|
414
439
|
}
|
|
415
|
-
}
|
|
440
|
+
};
|
|
416
441
|
}
|
|
417
442
|
async getPerformanceData() {
|
|
418
443
|
return {
|
|
@@ -421,19 +446,19 @@ class Le extends L {
|
|
|
421
446
|
};
|
|
422
447
|
}
|
|
423
448
|
}
|
|
424
|
-
const
|
|
425
|
-
var
|
|
426
|
-
const
|
|
427
|
-
Closed_Fist:
|
|
428
|
-
Open_Palm:
|
|
429
|
-
Pointing_Up:
|
|
430
|
-
Thumb_Down:
|
|
431
|
-
Thumb_Up:
|
|
432
|
-
Victory:
|
|
433
|
-
ILoveYou:
|
|
434
|
-
None:
|
|
435
|
-
},
|
|
436
|
-
class
|
|
449
|
+
const Je = (i) => i > 1 ? 1024 : 512;
|
|
450
|
+
var D = /* @__PURE__ */ ((i) => (i.ClosedFist = "closed_fist", i.OpenPalm = "open_palm", i.PointingUp = "pointing_up", i.ThumbDown = "thumb_down", i.ThumbUp = "thumb_up", i.Victory = "victory", i.ILoveYou = "i_love_you", i.None = "none", i))(D || {});
|
|
451
|
+
const et = 512, tt = {
|
|
452
|
+
Closed_Fist: D.ClosedFist,
|
|
453
|
+
Open_Palm: D.OpenPalm,
|
|
454
|
+
Pointing_Up: D.PointingUp,
|
|
455
|
+
Thumb_Down: D.ThumbDown,
|
|
456
|
+
Thumb_Up: D.ThumbUp,
|
|
457
|
+
Victory: D.Victory,
|
|
458
|
+
ILoveYou: D.ILoveYou,
|
|
459
|
+
None: D.None
|
|
460
|
+
}, st = 15;
|
|
461
|
+
class it extends N {
|
|
437
462
|
constructor() {
|
|
438
463
|
super();
|
|
439
464
|
h(this, "landmarker", null);
|
|
@@ -441,25 +466,25 @@ class qe extends L {
|
|
|
441
466
|
h(this, "coordinates");
|
|
442
467
|
h(this, "gesture");
|
|
443
468
|
h(this, "sourceCanvas");
|
|
444
|
-
h(this, "downscaler", new
|
|
469
|
+
h(this, "downscaler", new Z(et));
|
|
445
470
|
}
|
|
446
471
|
async getGestureData() {
|
|
447
472
|
if (!this.landmarker || !this.sourceCanvas)
|
|
448
473
|
return;
|
|
449
|
-
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(),
|
|
474
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), a = this.landmarker.recognizeForVideo(e, t), r = performance.now();
|
|
450
475
|
this.processingTime = r - t;
|
|
451
|
-
const
|
|
452
|
-
if (
|
|
453
|
-
const m =
|
|
454
|
-
return
|
|
455
|
-
|
|
456
|
-
x:
|
|
457
|
-
y:
|
|
458
|
-
z: -
|
|
476
|
+
const o = a.landmarks[0], u = [];
|
|
477
|
+
if (o) {
|
|
478
|
+
const m = a.gestures[0][0].categoryName;
|
|
479
|
+
return o.forEach((d) => {
|
|
480
|
+
u.push({
|
|
481
|
+
x: O(d.x),
|
|
482
|
+
y: O(1 - d.y),
|
|
483
|
+
z: -d.z
|
|
459
484
|
});
|
|
460
485
|
}), {
|
|
461
|
-
coordinates:
|
|
462
|
-
gesture:
|
|
486
|
+
coordinates: u,
|
|
487
|
+
gesture: tt[m]
|
|
463
488
|
};
|
|
464
489
|
}
|
|
465
490
|
return { coordinates: void 0, gesture: void 0 };
|
|
@@ -487,11 +512,11 @@ class qe extends L {
|
|
|
487
512
|
this.sourceCanvas = void 0, this.downscaler.dispose(), this.coordinates = void 0, this.gesture = void 0;
|
|
488
513
|
}
|
|
489
514
|
async initialize() {
|
|
490
|
-
const e = await
|
|
491
|
-
new URL("/assets/xr/wasm",
|
|
492
|
-
), t = await
|
|
493
|
-
new URL("/assets/xr/models/gesture_tracking.task",
|
|
494
|
-
),
|
|
515
|
+
const e = await V.forVisionTasks(
|
|
516
|
+
new URL("/assets/xr/wasm", I.assetStorageEndpoint).toString()
|
|
517
|
+
), t = await K(
|
|
518
|
+
new URL("/assets/xr/models/gesture_tracking.task", I.assetStorageEndpoint).toString()
|
|
519
|
+
), a = await Pe.createFromOptions(e, {
|
|
495
520
|
baseOptions: {
|
|
496
521
|
modelAssetBuffer: t,
|
|
497
522
|
delegate: "GPU"
|
|
@@ -499,10 +524,10 @@ class qe extends L {
|
|
|
499
524
|
runningMode: "VIDEO",
|
|
500
525
|
numHands: 1
|
|
501
526
|
});
|
|
502
|
-
this.landmarker =
|
|
527
|
+
this.landmarker = a;
|
|
503
528
|
}
|
|
504
529
|
async run({ sourceCanvas: e, targetFps: t }) {
|
|
505
|
-
this.sourceCanvas = e, this.setTargetFps(t ??
|
|
530
|
+
this.sourceCanvas = e, this.setTargetFps(t ?? st), this.rafId === null && await this.processGestureTracking();
|
|
506
531
|
}
|
|
507
532
|
getModelNode() {
|
|
508
533
|
const e = this;
|
|
@@ -522,43 +547,55 @@ class qe extends L {
|
|
|
522
547
|
};
|
|
523
548
|
}
|
|
524
549
|
}
|
|
525
|
-
const
|
|
526
|
-
class
|
|
550
|
+
const nt = 512;
|
|
551
|
+
class at extends N {
|
|
527
552
|
constructor() {
|
|
528
553
|
super();
|
|
529
554
|
h(this, "landmarker", null);
|
|
530
555
|
h(this, "processingTime", null);
|
|
556
|
+
h(this, "hands");
|
|
531
557
|
h(this, "coordinates");
|
|
532
558
|
h(this, "gesture");
|
|
559
|
+
h(this, "handedness");
|
|
533
560
|
h(this, "sourceCanvas");
|
|
534
|
-
h(this, "downscaler", new
|
|
561
|
+
h(this, "downscaler", new Z(nt));
|
|
535
562
|
}
|
|
536
563
|
async getHandData() {
|
|
537
564
|
if (!this.landmarker || !this.sourceCanvas)
|
|
538
565
|
return;
|
|
539
|
-
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(),
|
|
566
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), a = this.landmarker.detectForVideo(e, t), r = performance.now();
|
|
540
567
|
this.processingTime = r - t;
|
|
541
|
-
const
|
|
542
|
-
if (
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
}),
|
|
551
|
-
|
|
552
|
-
|
|
568
|
+
const o = [], u = a.handedness ?? a.handednesses ?? [];
|
|
569
|
+
if (a.landmarks.forEach((d, S) => {
|
|
570
|
+
var f, w, p;
|
|
571
|
+
if (!d || d.length === 0)
|
|
572
|
+
return;
|
|
573
|
+
const R = d.map((k) => ({
|
|
574
|
+
x: O(k.x),
|
|
575
|
+
y: O(1 - k.y),
|
|
576
|
+
z: -k.z
|
|
577
|
+
})), M = (p = (w = (f = u[S]) == null ? void 0 : f[0]) == null ? void 0 : w.categoryName) == null ? void 0 : p.toLowerCase(), x = M === "left" || M === "right" ? M : void 0, g = d[0].z < 0 ? "closed_fist" : "open_palm";
|
|
578
|
+
o.push({ coordinates: R, gesture: g, handedness: x });
|
|
579
|
+
}), !o.length)
|
|
580
|
+
return {
|
|
581
|
+
hands: void 0,
|
|
582
|
+
coordinates: void 0,
|
|
583
|
+
gesture: void 0,
|
|
584
|
+
handedness: void 0
|
|
553
585
|
};
|
|
554
|
-
|
|
555
|
-
return {
|
|
586
|
+
const m = o[0];
|
|
587
|
+
return {
|
|
588
|
+
hands: o,
|
|
589
|
+
coordinates: m.coordinates,
|
|
590
|
+
gesture: m.gesture,
|
|
591
|
+
handedness: m.handedness
|
|
592
|
+
};
|
|
556
593
|
}
|
|
557
594
|
async processHandTracking() {
|
|
558
595
|
if (!this.disposed) {
|
|
559
596
|
if (this.shouldRunInference()) {
|
|
560
597
|
const e = await this.getHandData();
|
|
561
|
-
e && (this.coordinates = e.coordinates, this.gesture = e.gesture);
|
|
598
|
+
e && (this.hands = e.hands, this.coordinates = e.coordinates, this.gesture = e.gesture, this.handedness = e.handedness);
|
|
562
599
|
}
|
|
563
600
|
this.disposed || this.paused || (this.rafId = requestAnimationFrame(this.processHandTracking.bind(this)));
|
|
564
601
|
}
|
|
@@ -574,22 +611,22 @@ class We extends L {
|
|
|
574
611
|
}
|
|
575
612
|
this.landmarker = null;
|
|
576
613
|
}
|
|
577
|
-
this.sourceCanvas = void 0, this.downscaler.dispose(), this.coordinates = void 0, this.gesture = void 0;
|
|
614
|
+
this.sourceCanvas = void 0, this.hands = void 0, this.downscaler.dispose(), this.coordinates = void 0, this.gesture = void 0, this.handedness = void 0;
|
|
578
615
|
}
|
|
579
616
|
async initialize() {
|
|
580
|
-
const e = await
|
|
581
|
-
new URL("/assets/xr/wasm",
|
|
582
|
-
), t = await
|
|
583
|
-
new URL("/assets/xr/models/hand_tracking.task",
|
|
584
|
-
),
|
|
617
|
+
const e = await V.forVisionTasks(
|
|
618
|
+
new URL("/assets/xr/wasm", I.assetStorageEndpoint).toString()
|
|
619
|
+
), t = await K(
|
|
620
|
+
new URL("/assets/xr/models/hand_tracking.task", I.assetStorageEndpoint).toString()
|
|
621
|
+
), a = await Oe.createFromOptions(e, {
|
|
585
622
|
baseOptions: {
|
|
586
623
|
modelAssetBuffer: t,
|
|
587
624
|
delegate: "GPU"
|
|
588
625
|
},
|
|
589
626
|
runningMode: "VIDEO",
|
|
590
|
-
numHands:
|
|
627
|
+
numHands: 2
|
|
591
628
|
});
|
|
592
|
-
this.landmarker =
|
|
629
|
+
this.landmarker = a;
|
|
593
630
|
}
|
|
594
631
|
async run({ sourceCanvas: e, targetFps: t }) {
|
|
595
632
|
this.sourceCanvas = e, this.setTargetFps(t), this.rafId === null && await this.processHandTracking();
|
|
@@ -597,11 +634,17 @@ class We extends L {
|
|
|
597
634
|
getModelNode() {
|
|
598
635
|
const e = this;
|
|
599
636
|
return {
|
|
637
|
+
get hands() {
|
|
638
|
+
return e.hands;
|
|
639
|
+
},
|
|
600
640
|
get coordinates() {
|
|
601
641
|
return e.coordinates;
|
|
602
642
|
},
|
|
603
643
|
get gesture() {
|
|
604
644
|
return e.gesture;
|
|
645
|
+
},
|
|
646
|
+
get handedness() {
|
|
647
|
+
return e.handedness;
|
|
605
648
|
}
|
|
606
649
|
};
|
|
607
650
|
}
|
|
@@ -612,13 +655,13 @@ class We extends L {
|
|
|
612
655
|
};
|
|
613
656
|
}
|
|
614
657
|
}
|
|
615
|
-
const
|
|
658
|
+
const rt = 24, ot = 720, ct = [
|
|
616
659
|
[255, 255, 255, 255],
|
|
617
660
|
// 0: background → green 255, kept
|
|
618
661
|
[0, 0, 0, 0]
|
|
619
662
|
// 1: person → green 0, discarded
|
|
620
663
|
];
|
|
621
|
-
class
|
|
664
|
+
class ht extends N {
|
|
622
665
|
constructor() {
|
|
623
666
|
super();
|
|
624
667
|
h(this, "segmenter", null);
|
|
@@ -690,8 +733,8 @@ class Ye extends L {
|
|
|
690
733
|
return e.videoWidth > 0 ? { width: e.videoWidth, height: e.videoHeight } : void 0;
|
|
691
734
|
if (e instanceof HTMLImageElement)
|
|
692
735
|
return e.naturalWidth > 0 ? { width: e.naturalWidth, height: e.naturalHeight } : void 0;
|
|
693
|
-
const t = e.width,
|
|
694
|
-
return typeof t == "number" && t > 0 && typeof
|
|
736
|
+
const t = e.width, a = e.height;
|
|
737
|
+
return typeof t == "number" && t > 0 && typeof a == "number" && a > 0 ? { width: t, height: a } : void 0;
|
|
695
738
|
}
|
|
696
739
|
/**
|
|
697
740
|
* Size the process canvas to the override input's aspect ratio, bounded by
|
|
@@ -704,8 +747,8 @@ class Ye extends L {
|
|
|
704
747
|
const e = this.readIntrinsicSize(this.inputSource);
|
|
705
748
|
if (!e) return;
|
|
706
749
|
this.inputSourceSize = e;
|
|
707
|
-
const t = Math.min(1,
|
|
708
|
-
this.processCanvas.width ===
|
|
750
|
+
const t = Math.min(1, ot / Math.max(e.width, e.height)), a = Math.max(1, Math.round(e.width * t)), r = Math.max(1, Math.round(e.height * t));
|
|
751
|
+
this.processCanvas.width === a && this.processCanvas.height === r || (this.processCanvas.width = a, this.processCanvas.height = r, this.glCanvas && (this.glCanvas.width = a, this.glCanvas.height = r), this.maskImageData = void 0);
|
|
709
752
|
}
|
|
710
753
|
async processSegmentation() {
|
|
711
754
|
if (this.disposed || !this.isProcessing || !this.maskCanvas || !this.processCanvas)
|
|
@@ -723,32 +766,32 @@ class Ye extends L {
|
|
|
723
766
|
}
|
|
724
767
|
const t = performance.now();
|
|
725
768
|
this.ensureProcessCanvasForInput();
|
|
726
|
-
const
|
|
727
|
-
if (
|
|
728
|
-
const
|
|
729
|
-
if (
|
|
730
|
-
const { width:
|
|
731
|
-
|
|
769
|
+
const a = this.processCanvas.getContext("2d");
|
|
770
|
+
if (a) {
|
|
771
|
+
const u = this.processCanvas.width, m = this.processCanvas.height;
|
|
772
|
+
if (a.clearRect(0, 0, u, m), this.inputSource && this.inputSourceSize) {
|
|
773
|
+
const { width: d, height: S } = this.inputSourceSize, R = Math.min(u / d, m / S), M = d * R, x = S * R;
|
|
774
|
+
a.drawImage(this.inputSource, (u - M) / 2, (m - x) / 2, M, x);
|
|
732
775
|
} else
|
|
733
|
-
|
|
776
|
+
a.drawImage(e, 0, 0, u, m);
|
|
734
777
|
}
|
|
735
|
-
const
|
|
778
|
+
const o = this.segmenter.segmentForVideo(
|
|
736
779
|
this.processCanvas,
|
|
737
780
|
performance.now()
|
|
738
781
|
).categoryMask;
|
|
739
|
-
if (!
|
|
782
|
+
if (!o) {
|
|
740
783
|
if (this.disposed || this.paused) return;
|
|
741
784
|
this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
742
785
|
return;
|
|
743
786
|
}
|
|
744
787
|
if (this.rawCache = null, this.latestMask && (this.latestMask.close(), this.latestMask = null), this.useGpuMaskPath && this.drawingUtils && this.glCanvas)
|
|
745
|
-
this.drawingUtils.drawCategoryMask(
|
|
788
|
+
this.drawingUtils.drawCategoryMask(o, ct), this.latestMask = o.clone(), o.close(), this.blitGlToMask();
|
|
746
789
|
else {
|
|
747
|
-
this.latestMask =
|
|
748
|
-
const
|
|
749
|
-
|
|
790
|
+
this.latestMask = o.clone();
|
|
791
|
+
const u = this.readMaskToImageData(o);
|
|
792
|
+
o.close(), u && this.blitImageDataToMask(u);
|
|
750
793
|
}
|
|
751
|
-
this.segmentationMaskTexture && (this.isMaskSetup || (this.segmentationMaskTexture.format =
|
|
794
|
+
this.segmentationMaskTexture && (this.isMaskSetup || (this.segmentationMaskTexture.format = de, this.segmentationMaskTexture.minFilter = q, this.segmentationMaskTexture.magFilter = q, this.segmentationMaskTexture.generateMipmaps = !1, this.isMaskSetup = !0), this.segmentationMaskTexture.needsUpdate = !0), this.processingTime = performance.now() - t, !(this.disposed || this.paused) && (this.rafId = requestAnimationFrame(this.processSegmentation.bind(this)));
|
|
752
795
|
}
|
|
753
796
|
/**
|
|
754
797
|
* Blit the GPU-rendered mask (already RGBA-inverted by drawCategoryMask)
|
|
@@ -768,13 +811,13 @@ class Ye extends L {
|
|
|
768
811
|
* background→opaque convention (green carries the signal for the alphaMap).
|
|
769
812
|
*/
|
|
770
813
|
readMaskToImageData(e) {
|
|
771
|
-
const t = e.getAsUint8Array(), { width:
|
|
772
|
-
if (!t ||
|
|
773
|
-
(!this.maskImageData || this.maskImageData.width !==
|
|
774
|
-
const
|
|
775
|
-
for (let
|
|
776
|
-
const m = t[
|
|
777
|
-
|
|
814
|
+
const t = e.getAsUint8Array(), { width: a, height: r } = e;
|
|
815
|
+
if (!t || a <= 0 || r <= 0) return null;
|
|
816
|
+
(!this.maskImageData || this.maskImageData.width !== a || this.maskImageData.height !== r) && (this.maskImageData = new ImageData(a, r));
|
|
817
|
+
const o = this.maskImageData.data;
|
|
818
|
+
for (let u = 0; u < t.length; u += 1) {
|
|
819
|
+
const m = t[u] === 255 ? 0 : 255;
|
|
820
|
+
o[u * 4] = 255, o[u * 4 + 1] = m, o[u * 4 + 2] = 255, o[u * 4 + 3] = m;
|
|
778
821
|
}
|
|
779
822
|
return this.maskImageData;
|
|
780
823
|
}
|
|
@@ -785,10 +828,10 @@ class Ye extends L {
|
|
|
785
828
|
*/
|
|
786
829
|
blitImageDataToMask(e) {
|
|
787
830
|
if (!this.maskCanvas || !this.filterCanvas) return;
|
|
788
|
-
const t = this.maskCanvas.getContext("2d"),
|
|
789
|
-
if (!t || !
|
|
790
|
-
const r = this.getStagingCanvas(e.width, e.height),
|
|
791
|
-
|
|
831
|
+
const t = this.maskCanvas.getContext("2d"), a = this.filterCanvas.getContext("2d");
|
|
832
|
+
if (!t || !a) return;
|
|
833
|
+
const r = this.getStagingCanvas(e.width, e.height), o = r.getContext("2d");
|
|
834
|
+
o && (o.putImageData(e, 0, 0), a.filter = "blur(0.5px)", a.clearRect(0, 0, this.filterCanvas.width, this.filterCanvas.height), a.drawImage(r, 0, 0, this.filterCanvas.width, this.filterCanvas.height), t.clearRect(0, 0, this.maskCanvas.width, this.maskCanvas.height), t.drawImage(this.filterCanvas, 0, 0, this.maskCanvas.width, this.maskCanvas.height));
|
|
792
835
|
}
|
|
793
836
|
getStagingCanvas(e, t) {
|
|
794
837
|
return this.stagingCanvas || (this.stagingCanvas = document.createElement("canvas")), this.stagingCanvas.width !== e && (this.stagingCanvas.width = e), this.stagingCanvas.height !== t && (this.stagingCanvas.height = t), this.stagingCanvas;
|
|
@@ -815,37 +858,38 @@ class Ye extends L {
|
|
|
815
858
|
}
|
|
816
859
|
async initialize(e) {
|
|
817
860
|
this.quality = (e == null ? void 0 : e.quality) ?? "low";
|
|
818
|
-
const t = await
|
|
819
|
-
new URL("/assets/xr/wasm",
|
|
820
|
-
),
|
|
821
|
-
new URL("/assets/xr/models/segmentation.tflite",
|
|
861
|
+
const t = await V.forVisionTasks(
|
|
862
|
+
new URL("/assets/xr/wasm", I.assetStorageEndpoint).toString()
|
|
863
|
+
), a = await K(
|
|
864
|
+
new URL("/assets/xr/models/segmentation.tflite", I.assetStorageEndpoint).toString()
|
|
822
865
|
);
|
|
823
866
|
this.glCanvas = new OffscreenCanvas(1, 1);
|
|
824
867
|
const r = this.glCanvas.getContext("webgl2", {
|
|
825
868
|
preserveDrawingBuffer: !0
|
|
826
869
|
});
|
|
827
|
-
this.segmenter = await
|
|
870
|
+
this.segmenter = await Ue.createFromOptions(t, {
|
|
828
871
|
baseOptions: {
|
|
829
|
-
modelAssetBuffer:
|
|
872
|
+
modelAssetBuffer: a,
|
|
830
873
|
delegate: "GPU"
|
|
831
874
|
},
|
|
832
875
|
...r ? { canvas: this.glCanvas } : {},
|
|
833
876
|
outputCategoryMask: !0,
|
|
834
877
|
outputConfidenceMasks: !1,
|
|
835
878
|
runningMode: "VIDEO"
|
|
836
|
-
}), r ? (this.drawingUtils = new
|
|
879
|
+
}), r ? (this.drawingUtils = new ze(r), this.useGpuMaskPath = !0) : this.useGpuMaskPath = !1;
|
|
837
880
|
}
|
|
838
881
|
async run({
|
|
839
882
|
sourceCanvas: e,
|
|
840
883
|
renderWidth: t,
|
|
841
|
-
renderHeight:
|
|
884
|
+
renderHeight: a,
|
|
842
885
|
targetFps: r
|
|
843
886
|
}) {
|
|
887
|
+
var u;
|
|
844
888
|
if (!e)
|
|
845
889
|
return;
|
|
846
|
-
this.setTargetFps(r ??
|
|
847
|
-
const
|
|
848
|
-
this.sourceCanvas = e, this.processCanvas = document.createElement("canvas"), this.processCanvas.width = e.width *
|
|
890
|
+
this.setTargetFps(r ?? rt), this.isProcessing = !1, this.rafId !== null && (cancelAnimationFrame(this.rafId), this.rafId = null);
|
|
891
|
+
const o = this.getQualityScaleFactor();
|
|
892
|
+
this.sourceCanvas = e, this.processCanvas = document.createElement("canvas"), this.processCanvas.width = e.width * o, this.processCanvas.height = e.height * o, this.glCanvas && (this.glCanvas.width = this.processCanvas.width, this.glCanvas.height = this.processCanvas.height), this.maskImageData = void 0, this.maskCanvas = document.createElement("canvas"), this.maskCanvas.width = t, this.maskCanvas.height = a, this.filterCanvas = document.createElement("canvas"), this.filterCanvas.width = t, this.filterCanvas.height = a, (u = this.segmentationMaskTexture) == null || u.dispose(), this.segmentationMaskTexture = new ue(this.maskCanvas), this.isMaskSetup = !1, this.rawCache = null, this.isProcessing = !0, this.lastInferenceMs = 0, this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
849
893
|
}
|
|
850
894
|
/**
|
|
851
895
|
* Lazily read the single-channel mask back to a Uint8Array. Only invoked when
|
|
@@ -880,13 +924,13 @@ class Ye extends L {
|
|
|
880
924
|
};
|
|
881
925
|
}
|
|
882
926
|
}
|
|
883
|
-
const
|
|
884
|
-
[
|
|
885
|
-
[
|
|
886
|
-
[
|
|
887
|
-
[
|
|
888
|
-
[
|
|
889
|
-
},
|
|
927
|
+
const re = {
|
|
928
|
+
[z.BACKGROUND_SEGMENTATION]: ht,
|
|
929
|
+
[z.HAND_TRACKER]: at,
|
|
930
|
+
[z.GESTURE_TRACKER]: it,
|
|
931
|
+
[z.FACE_TRACKER]: Ze,
|
|
932
|
+
[z.BODY_TRACKER]: je
|
|
933
|
+
}, dt = async (i) => {
|
|
890
934
|
const s = document.createElement("video");
|
|
891
935
|
s.srcObject = i, s.muted = !0, s.playsInline = !0, s.autoplay = !0, await new Promise((t) => {
|
|
892
936
|
s.onloadedmetadata = () => {
|
|
@@ -895,43 +939,52 @@ const Z = {
|
|
|
895
939
|
});
|
|
896
940
|
const e = s;
|
|
897
941
|
return await new Promise((t) => {
|
|
898
|
-
const
|
|
899
|
-
clearTimeout(
|
|
942
|
+
const a = setTimeout(t, 150), r = () => {
|
|
943
|
+
clearTimeout(a), t();
|
|
900
944
|
};
|
|
901
945
|
typeof e.requestVideoFrameCallback == "function" ? e.requestVideoFrameCallback(r) : requestAnimationFrame(() => requestAnimationFrame(() => r()));
|
|
902
946
|
}), s;
|
|
903
|
-
},
|
|
904
|
-
const s = new
|
|
905
|
-
return s.colorSpace =
|
|
906
|
-
},
|
|
947
|
+
}, ut = (i) => {
|
|
948
|
+
const s = new ue(i);
|
|
949
|
+
return s.colorSpace = Le, s.format = de, s.minFilter = q, s.magFilter = q, s.generateMipmaps = !1, s.needsUpdate = !0, s;
|
|
950
|
+
}, lt = (i, s) => {
|
|
907
951
|
const e = i.videoWidth, t = i.videoHeight;
|
|
908
952
|
e > 0 && t > 0 && (s.width !== e && (s.width = e), s.height !== t && (s.height = t));
|
|
909
|
-
const
|
|
953
|
+
const a = s.getContext("2d", {
|
|
910
954
|
colorSpace: "srgb"
|
|
911
955
|
});
|
|
912
|
-
|
|
913
|
-
},
|
|
956
|
+
a.save(), a.scale(-1, 1), a.drawImage(i, -s.width, 0, s.width, s.height), a.restore();
|
|
957
|
+
}, gt = (i, s, e = 1) => {
|
|
914
958
|
const t = s.getContext("2d", {
|
|
915
959
|
colorSpace: "srgb"
|
|
916
|
-
}),
|
|
917
|
-
let
|
|
918
|
-
r >
|
|
960
|
+
}), a = s.width / s.height, r = i.width / i.height;
|
|
961
|
+
let o, u;
|
|
962
|
+
r > a ? (u = s.height, o = i.width * (s.height / i.height)) : (o = s.width, u = i.height * (s.width / i.width));
|
|
919
963
|
const m = Number.isFinite(e) && e > 0 ? e : 1;
|
|
920
|
-
|
|
921
|
-
const
|
|
964
|
+
o *= m, u *= m;
|
|
965
|
+
const d = (s.width - o) / 2, S = (s.height - u) / 2;
|
|
922
966
|
t.setTransform(1, 0, 0, 1, 0, 0), t.clearRect(0, 0, s.width, s.height), t.drawImage(
|
|
923
967
|
i,
|
|
924
968
|
0,
|
|
925
969
|
0,
|
|
926
970
|
i.width,
|
|
927
971
|
i.height,
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
972
|
+
d,
|
|
973
|
+
S,
|
|
974
|
+
o,
|
|
975
|
+
u
|
|
932
976
|
);
|
|
933
|
-
},
|
|
977
|
+
}, mt = (i, s, e) => {
|
|
934
978
|
const t = /* @__PURE__ */ new Map();
|
|
979
|
+
let a = !1;
|
|
980
|
+
const r = (g, f) => {
|
|
981
|
+
var w;
|
|
982
|
+
try {
|
|
983
|
+
(w = f.dispose) == null || w.call(f);
|
|
984
|
+
} catch (p) {
|
|
985
|
+
console.warn(`[xr-sdk registry] dispose threw for ${g}`, p);
|
|
986
|
+
}
|
|
987
|
+
};
|
|
935
988
|
return { acquire: async (g, f, w) => {
|
|
936
989
|
let p = t.get(g);
|
|
937
990
|
if (p)
|
|
@@ -939,33 +992,36 @@ const Z = {
|
|
|
939
992
|
const k = s[g];
|
|
940
993
|
if (!k)
|
|
941
994
|
throw new Error(`Invalid model: ${g}`);
|
|
942
|
-
const C = new k(),
|
|
943
|
-
await C.initialize(w)
|
|
944
|
-
const
|
|
945
|
-
|
|
995
|
+
const C = new k(), L = /* @__PURE__ */ new Set([f]), U = /* @__PURE__ */ new Set([f]), _ = (async () => {
|
|
996
|
+
await C.initialize(w);
|
|
997
|
+
const B = t.get(g);
|
|
998
|
+
if (!B || B.model !== C || B.refs.size === 0)
|
|
999
|
+
return r(g, C), C;
|
|
1000
|
+
e && await e(g, C);
|
|
1001
|
+
const T = t.get(g);
|
|
1002
|
+
return T && T.refs.size > 0 && i.setPipeline((P) => P.set(g, C)), C;
|
|
946
1003
|
})();
|
|
947
|
-
return p = { model: C, refs:
|
|
948
|
-
t.get(g) === p && (t.delete(g), i.setPipeline((
|
|
949
|
-
}),
|
|
1004
|
+
return p = { model: C, refs: L, activeRefs: U, initPromise: _, options: w }, t.set(g, p), _.catch(() => {
|
|
1005
|
+
t.get(g) === p && (t.delete(g), i.setPipeline((T) => T.delete(g))), r(g, C);
|
|
1006
|
+
}), _;
|
|
950
1007
|
}, release: (g, f) => {
|
|
951
|
-
var k;
|
|
952
1008
|
const w = t.get(g);
|
|
953
|
-
|
|
954
|
-
t.delete(g);
|
|
955
|
-
const p = w.model;
|
|
956
|
-
try {
|
|
957
|
-
(k = p.dispose) == null || k.call(p);
|
|
958
|
-
} catch (C) {
|
|
959
|
-
console.warn(`[xr-sdk registry] dispose threw for ${g}`, C);
|
|
960
|
-
}
|
|
961
|
-
i.setPipeline((C) => C.delete(g));
|
|
1009
|
+
w && (w.refs.delete(f), w.activeRefs.delete(f) && w.refs.size > 0 && w.activeRefs.size === 0 && w.model.pause(), !(w.refs.size > 0) && (t.delete(g), r(g, w.model), i.setPipeline((p) => p.delete(g))));
|
|
962
1010
|
}, setActive: (g, f, w) => {
|
|
963
1011
|
const p = t.get(g);
|
|
964
1012
|
if (!p || !p.refs.has(f)) return;
|
|
965
1013
|
const k = p.activeRefs.size > 0;
|
|
966
1014
|
w ? p.activeRefs.add(f) : p.activeRefs.delete(f);
|
|
967
1015
|
const C = p.activeRefs.size > 0;
|
|
968
|
-
k !== C && (C ? p.model.resume() : p.model.pause());
|
|
1016
|
+
k !== C && (C && !a ? p.model.resume() : p.model.pause());
|
|
1017
|
+
}, suspendAll: () => {
|
|
1018
|
+
a = !0;
|
|
1019
|
+
for (const g of t.values())
|
|
1020
|
+
g.model.pause();
|
|
1021
|
+
}, resumeActive: () => {
|
|
1022
|
+
a = !1;
|
|
1023
|
+
for (const g of t.values())
|
|
1024
|
+
g.activeRefs.size > 0 && g.model.resume();
|
|
969
1025
|
}, get: (g) => {
|
|
970
1026
|
var f;
|
|
971
1027
|
return (f = t.get(g)) == null ? void 0 : f.model;
|
|
@@ -973,21 +1029,13 @@ const Z = {
|
|
|
973
1029
|
var f;
|
|
974
1030
|
return (f = t.get(g)) == null ? void 0 : f.options;
|
|
975
1031
|
}, releaseAll: () => {
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
if (!w) continue;
|
|
980
|
-
w.refs.clear(), t.delete(f);
|
|
981
|
-
const p = w.model;
|
|
982
|
-
try {
|
|
983
|
-
(g = p.dispose) == null || g.call(p);
|
|
984
|
-
} catch (k) {
|
|
985
|
-
console.warn(`[xr-sdk registry] dispose threw for ${f}`, k);
|
|
986
|
-
}
|
|
1032
|
+
for (const g of Array.from(t.keys())) {
|
|
1033
|
+
const f = t.get(g);
|
|
1034
|
+
f && (f.refs.clear(), t.delete(g), r(g, f.model));
|
|
987
1035
|
}
|
|
988
|
-
i.setPipeline((
|
|
1036
|
+
i.setPipeline((g) => g.clear());
|
|
989
1037
|
} };
|
|
990
|
-
},
|
|
1038
|
+
}, ft = {
|
|
991
1039
|
lifecycle: "idle",
|
|
992
1040
|
ready: !1,
|
|
993
1041
|
error: void 0,
|
|
@@ -997,31 +1045,31 @@ const Z = {
|
|
|
997
1045
|
source: void 0,
|
|
998
1046
|
sourceKind: void 0,
|
|
999
1047
|
pipeline: /* @__PURE__ */ new Map()
|
|
1000
|
-
},
|
|
1001
|
-
let i = { ...
|
|
1048
|
+
}, pt = () => {
|
|
1049
|
+
let i = { ...ft, pipeline: /* @__PURE__ */ new Map() };
|
|
1002
1050
|
const s = /* @__PURE__ */ new Set(), e = () => {
|
|
1003
1051
|
s.forEach((t) => t());
|
|
1004
1052
|
};
|
|
1005
1053
|
return {
|
|
1006
1054
|
getSnapshot: () => i,
|
|
1007
1055
|
set: (t) => {
|
|
1008
|
-
let
|
|
1056
|
+
let a = !1;
|
|
1009
1057
|
for (const r of Object.keys(t))
|
|
1010
1058
|
if (t[r] !== i[r]) {
|
|
1011
|
-
|
|
1059
|
+
a = !0;
|
|
1012
1060
|
break;
|
|
1013
1061
|
}
|
|
1014
|
-
|
|
1062
|
+
a && (i = { ...i, ...t }, e());
|
|
1015
1063
|
},
|
|
1016
1064
|
setPipeline: (t) => {
|
|
1017
|
-
const
|
|
1018
|
-
return t(
|
|
1065
|
+
const a = new Map(i.pipeline);
|
|
1066
|
+
return t(a), i = { ...i, pipeline: a }, e(), a;
|
|
1019
1067
|
},
|
|
1020
1068
|
subscribe: (t) => (s.add(t), () => {
|
|
1021
1069
|
s.delete(t);
|
|
1022
1070
|
})
|
|
1023
1071
|
};
|
|
1024
|
-
},
|
|
1072
|
+
}, vt = ({
|
|
1025
1073
|
renderHeight: i,
|
|
1026
1074
|
renderWidth: s,
|
|
1027
1075
|
videoHeight: e,
|
|
@@ -1029,303 +1077,322 @@ const Z = {
|
|
|
1029
1077
|
}) => ({
|
|
1030
1078
|
height: i || e,
|
|
1031
1079
|
width: s || t
|
|
1032
|
-
}),
|
|
1033
|
-
const s = new
|
|
1080
|
+
}), wt = (i) => {
|
|
1081
|
+
const s = new Ve({ verbose: !0 }), e = pt();
|
|
1034
1082
|
let t = i;
|
|
1035
|
-
const
|
|
1083
|
+
const a = (n) => {
|
|
1036
1084
|
var l;
|
|
1037
|
-
const
|
|
1038
|
-
return typeof
|
|
1085
|
+
const c = (l = T.getOptions(n)) == null ? void 0 : l.targetFps;
|
|
1086
|
+
return typeof c == "number" ? c : t == null ? void 0 : t.targetFps;
|
|
1039
1087
|
};
|
|
1040
|
-
let r,
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1088
|
+
let r, o, u, m, d, S, R, M, x, g, f, w = !1, p = !1, k = 0;
|
|
1089
|
+
const C = /* @__PURE__ */ new Set(), L = /* @__PURE__ */ new Set(), U = (n) => {
|
|
1090
|
+
const c = e.getSnapshot().cameraTexture;
|
|
1091
|
+
c && c.dispose();
|
|
1092
|
+
const l = ut(n);
|
|
1093
|
+
return e.set({ cameraTexture: l }), l;
|
|
1094
|
+
}, _ = async (n, c) => {
|
|
1095
|
+
if (C.has(n) || e.getSnapshot().lifecycle !== "running" || !d) return;
|
|
1096
|
+
C.add(n);
|
|
1097
|
+
const l = k;
|
|
1098
|
+
try {
|
|
1099
|
+
await c.run({
|
|
1100
|
+
sourceCanvas: d,
|
|
1101
|
+
renderWidth: e.getSnapshot().renderWidth,
|
|
1102
|
+
renderHeight: e.getSnapshot().renderHeight,
|
|
1103
|
+
getSource: () => e.getSnapshot().sourceKind,
|
|
1104
|
+
targetFps: a(n)
|
|
1105
|
+
}), p && (await new Promise((v) => requestAnimationFrame(() => v())), await new Promise((v) => requestAnimationFrame(() => v())), p && k === l && T.suspendAll());
|
|
1106
|
+
} catch (v) {
|
|
1107
|
+
throw C.delete(n), v;
|
|
1055
1108
|
}
|
|
1056
|
-
},
|
|
1109
|
+
}, T = mt(e, re, (n, c) => _(n, c));
|
|
1057
1110
|
e.subscribe(() => {
|
|
1058
1111
|
if (e.getSnapshot().lifecycle !== "running") return;
|
|
1059
|
-
const
|
|
1060
|
-
for (const [
|
|
1061
|
-
|
|
1112
|
+
const n = e.getSnapshot().pipeline;
|
|
1113
|
+
for (const [c, l] of n.entries())
|
|
1114
|
+
C.has(c) || _(c, l);
|
|
1062
1115
|
});
|
|
1063
|
-
const
|
|
1064
|
-
if (
|
|
1116
|
+
const P = () => {
|
|
1117
|
+
if (f !== void 0 && (cancelAnimationFrame(f), f = void 0), o) {
|
|
1065
1118
|
try {
|
|
1066
|
-
|
|
1119
|
+
o.pause();
|
|
1067
1120
|
} catch {
|
|
1068
1121
|
}
|
|
1069
1122
|
try {
|
|
1070
|
-
|
|
1123
|
+
o.srcObject = null;
|
|
1071
1124
|
} catch {
|
|
1072
1125
|
}
|
|
1073
|
-
|
|
1126
|
+
o = void 0;
|
|
1074
1127
|
}
|
|
1075
|
-
if (
|
|
1128
|
+
if (u) {
|
|
1076
1129
|
try {
|
|
1077
|
-
|
|
1130
|
+
u.getTracks().forEach((n) => n.stop());
|
|
1078
1131
|
} catch {
|
|
1079
1132
|
}
|
|
1080
|
-
|
|
1133
|
+
u = void 0;
|
|
1081
1134
|
}
|
|
1082
|
-
},
|
|
1083
|
-
if (
|
|
1084
|
-
const
|
|
1085
|
-
if (
|
|
1086
|
-
|
|
1135
|
+
}, G = () => {
|
|
1136
|
+
if (w || !o || !m || !d) return;
|
|
1137
|
+
const n = window.__xr_debugger_metrics;
|
|
1138
|
+
if (n != null && n.trackCameraFrame && n.trackCameraFrame(), (e.getSnapshot().sourceKind ?? "webcam") === "webcam")
|
|
1139
|
+
lt(o, m);
|
|
1087
1140
|
else {
|
|
1088
1141
|
const v = m.getContext("2d");
|
|
1089
1142
|
if (!v) return;
|
|
1090
|
-
const
|
|
1091
|
-
m.width !==
|
|
1143
|
+
const y = o.videoWidth, A = o.videoHeight;
|
|
1144
|
+
m.width !== y && (m.width = y), m.height !== A && (m.height = A), v.setTransform(1, 0, 0, 1, 0, 0), v.clearRect(0, 0, y, A), v.drawImage(o, 0, 0, y, A);
|
|
1092
1145
|
}
|
|
1093
|
-
|
|
1146
|
+
gt(m, d, (t == null ? void 0 : t.cameraZoom) ?? 1.2);
|
|
1094
1147
|
const l = e.getSnapshot().cameraTexture;
|
|
1095
1148
|
l && (l.needsUpdate = !0);
|
|
1096
|
-
},
|
|
1097
|
-
if (!
|
|
1098
|
-
if (
|
|
1099
|
-
|
|
1149
|
+
}, J = () => {
|
|
1150
|
+
if (!o || !m || !d) return;
|
|
1151
|
+
if (p) {
|
|
1152
|
+
G();
|
|
1100
1153
|
return;
|
|
1101
1154
|
}
|
|
1102
|
-
const
|
|
1103
|
-
|
|
1155
|
+
const n = () => {
|
|
1156
|
+
f = requestAnimationFrame(n), G();
|
|
1104
1157
|
};
|
|
1105
|
-
|
|
1106
|
-
},
|
|
1107
|
-
if (
|
|
1158
|
+
f = requestAnimationFrame(n);
|
|
1159
|
+
}, pe = async (n) => {
|
|
1160
|
+
if (n.source === F.WEBCAM)
|
|
1108
1161
|
try {
|
|
1109
|
-
const
|
|
1162
|
+
const c = t != null && t.camRes && t.camRes > 0 ? t.camRes : 960;
|
|
1110
1163
|
return await navigator.mediaDevices.getUserMedia({
|
|
1111
1164
|
video: {
|
|
1112
1165
|
facingMode: "user",
|
|
1113
1166
|
frameRate: { min: 24, ideal: 30, max: 60 },
|
|
1114
|
-
width: { ideal:
|
|
1167
|
+
width: { ideal: c }
|
|
1115
1168
|
}
|
|
1116
1169
|
});
|
|
1117
|
-
} catch (
|
|
1118
|
-
const l =
|
|
1119
|
-
throw e.set({ error: { error: "PermissionError", message: v } }), new
|
|
1170
|
+
} catch (c) {
|
|
1171
|
+
const l = c instanceof DOMException ? c.name : "", v = l === "NotReadableError" || l === "TrackStartError" ? "Camera is already in use by another app or browser tab. Close it and refresh." : l === "NotFoundError" || l === "DevicesNotFoundError" ? "No camera was found on this device." : l === "NotAllowedError" || l === "PermissionDeniedError" ? "Camera access was blocked. Allow camera permissions in your browser, then refresh." : "Unable to access camera. Allow camera permissions in your browser, then refresh.";
|
|
1172
|
+
throw e.set({ error: { error: "PermissionError", message: v } }), new b(c instanceof Error ? c.message : "getUserMedia failed");
|
|
1120
1173
|
}
|
|
1121
|
-
return
|
|
1122
|
-
},
|
|
1123
|
-
const
|
|
1124
|
-
|
|
1125
|
-
const l = await
|
|
1126
|
-
|
|
1127
|
-
const { renderWidth: v, renderHeight:
|
|
1174
|
+
return n.stream;
|
|
1175
|
+
}, W = async (n) => {
|
|
1176
|
+
const c = await pe(n);
|
|
1177
|
+
u = c;
|
|
1178
|
+
const l = await dt(c);
|
|
1179
|
+
o = l, m = new OffscreenCanvas(l.videoWidth, l.videoHeight);
|
|
1180
|
+
const { renderWidth: v, renderHeight: y } = e.getSnapshot(), A = vt({
|
|
1128
1181
|
renderWidth: v,
|
|
1129
|
-
renderHeight:
|
|
1182
|
+
renderHeight: y,
|
|
1130
1183
|
videoWidth: l.videoWidth,
|
|
1131
1184
|
videoHeight: l.videoHeight
|
|
1132
1185
|
});
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
const j = a.source === y.STREAM ? a.static === !0 : !1;
|
|
1141
|
-
R = j, r = a.source === y.WEBCAM ? { source: y.WEBCAM, sourceKind: "webcam" } : {
|
|
1142
|
-
source: y.STREAM,
|
|
1143
|
-
stream: a.stream,
|
|
1144
|
-
static: j,
|
|
1145
|
-
sourceKind: a.sourceKind ?? "webcam"
|
|
1186
|
+
d ? (d.width !== A.width || d.height !== A.height) && (d.width = A.width, d.height = A.height, U(d)) : (d = new OffscreenCanvas(A.width, A.height), U(d));
|
|
1187
|
+
const ie = n.source === F.STREAM ? n.static === !0 : !1;
|
|
1188
|
+
p = ie, r = n.source === F.WEBCAM ? { source: F.WEBCAM, sourceKind: "webcam" } : {
|
|
1189
|
+
source: F.STREAM,
|
|
1190
|
+
stream: n.stream,
|
|
1191
|
+
static: ie,
|
|
1192
|
+
sourceKind: n.sourceKind ?? "webcam"
|
|
1146
1193
|
}, e.set({
|
|
1147
|
-
source:
|
|
1194
|
+
source: n.source,
|
|
1148
1195
|
sourceKind: r.sourceKind,
|
|
1149
1196
|
error: void 0
|
|
1150
1197
|
});
|
|
1151
|
-
},
|
|
1152
|
-
const
|
|
1153
|
-
|
|
1154
|
-
const l =
|
|
1198
|
+
}, X = async (n) => {
|
|
1199
|
+
const c = n.sourceKind ?? "webcam";
|
|
1200
|
+
d = n.canvas;
|
|
1201
|
+
const l = U(d);
|
|
1155
1202
|
e.set({
|
|
1156
1203
|
cameraTexture: l,
|
|
1157
|
-
source:
|
|
1158
|
-
sourceKind:
|
|
1159
|
-
renderWidth:
|
|
1160
|
-
renderHeight:
|
|
1204
|
+
source: F.CANVAS,
|
|
1205
|
+
sourceKind: c,
|
|
1206
|
+
renderWidth: d.width,
|
|
1207
|
+
renderHeight: d.height,
|
|
1161
1208
|
error: void 0
|
|
1162
1209
|
}), r = {
|
|
1163
|
-
source:
|
|
1164
|
-
canvas:
|
|
1165
|
-
sourceKind:
|
|
1210
|
+
source: F.CANVAS,
|
|
1211
|
+
canvas: n.canvas,
|
|
1212
|
+
sourceKind: c
|
|
1166
1213
|
};
|
|
1167
|
-
},
|
|
1168
|
-
if (!
|
|
1169
|
-
const
|
|
1170
|
-
for (const [,
|
|
1171
|
-
|
|
1172
|
-
await new Promise((
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
if (
|
|
1178
|
-
throw new F(`Invalid media source: ${String(o)}`);
|
|
1179
|
-
if (ne(a, r)) {
|
|
1214
|
+
}, ve = (n, c) => !c || n.source !== c.source ? !1 : n.source === F.STREAM && c.source === F.STREAM ? n.stream === c.stream && (n.sourceKind ?? "webcam") === c.sourceKind : !0, we = async () => {
|
|
1215
|
+
if (!d) return;
|
|
1216
|
+
const n = k, c = e.getSnapshot().pipeline;
|
|
1217
|
+
for (const [, l] of c.entries())
|
|
1218
|
+
l.resume();
|
|
1219
|
+
await new Promise((l) => requestAnimationFrame(() => l())), await new Promise((l) => requestAnimationFrame(() => l())), p && k === n && T.suspendAll();
|
|
1220
|
+
}, Ce = (n, c) => !c || c.source !== F.CANVAS ? !1 : n.canvas === c.canvas && (n.sourceKind ?? "webcam") === c.sourceKind, Se = async (n) => {
|
|
1221
|
+
const c = n.source;
|
|
1222
|
+
if (c !== F.WEBCAM && c !== F.STREAM)
|
|
1223
|
+
throw new b(`Invalid media source: ${String(c)}`);
|
|
1224
|
+
if (ve(n, r)) {
|
|
1180
1225
|
s.log("Media source unchanged, skipping rebind.");
|
|
1181
1226
|
return;
|
|
1182
1227
|
}
|
|
1228
|
+
k += 1;
|
|
1183
1229
|
const { lifecycle: l } = e.getSnapshot();
|
|
1184
1230
|
if (l === "idle" || l === "stopped") {
|
|
1185
|
-
await
|
|
1231
|
+
await W(n), e.set({ lifecycle: "media-bound" }), s.log("Media source set to:", n.source);
|
|
1186
1232
|
return;
|
|
1187
1233
|
}
|
|
1188
1234
|
if (l === "media-bound") {
|
|
1189
|
-
|
|
1235
|
+
P(), await W(n), s.log("Media source rebound to:", n.source);
|
|
1190
1236
|
return;
|
|
1191
1237
|
}
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
const v = e.getSnapshot().pipeline;
|
|
1196
|
-
for (const [, S] of v.entries()) S.resume();
|
|
1197
|
-
}
|
|
1198
|
-
s.log("Media source swapped to:", a.source);
|
|
1199
|
-
}, he = async (a) => {
|
|
1200
|
-
if (oe(a, r)) {
|
|
1238
|
+
w = !0, await new Promise((v) => requestAnimationFrame(() => v())), P(), await W(n), w = !1, J(), p ? await we() : T.resumeActive(), s.log("Media source swapped to:", n.source);
|
|
1239
|
+
}, ke = async (n) => {
|
|
1240
|
+
if (Ce(n, r)) {
|
|
1201
1241
|
s.log("Frame source unchanged, skipping rebind.");
|
|
1202
1242
|
return;
|
|
1203
1243
|
}
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1244
|
+
k += 1;
|
|
1245
|
+
const { lifecycle: c } = e.getSnapshot();
|
|
1246
|
+
if (c === "idle" || c === "stopped") {
|
|
1247
|
+
await X(n), e.set({ lifecycle: "media-bound" }), s.log("Frame source set to canvas.");
|
|
1207
1248
|
return;
|
|
1208
1249
|
}
|
|
1209
|
-
if (
|
|
1210
|
-
|
|
1250
|
+
if (c === "media-bound") {
|
|
1251
|
+
P(), await X(n), s.log("Frame source rebound to canvas.");
|
|
1211
1252
|
return;
|
|
1212
1253
|
}
|
|
1213
|
-
|
|
1214
|
-
},
|
|
1215
|
-
const { lifecycle:
|
|
1216
|
-
if (
|
|
1217
|
-
if (
|
|
1218
|
-
throw new
|
|
1219
|
-
if (!
|
|
1220
|
-
throw new
|
|
1221
|
-
const
|
|
1222
|
-
if (!
|
|
1223
|
-
throw
|
|
1254
|
+
w = !0, await new Promise((l) => requestAnimationFrame(() => l())), P(), await X(n), w = !1, s.log("Frame source swapped to canvas.");
|
|
1255
|
+
}, ye = async () => {
|
|
1256
|
+
const { lifecycle: n } = e.getSnapshot();
|
|
1257
|
+
if (n === "running") return;
|
|
1258
|
+
if (n === "idle")
|
|
1259
|
+
throw new b("session.start() called before setMediaSource — call setMediaSource first");
|
|
1260
|
+
if (!d)
|
|
1261
|
+
throw new b("session.start(): media source not bound");
|
|
1262
|
+
const c = (r == null ? void 0 : r.source) === F.CANVAS;
|
|
1263
|
+
if (!c && (!o || !m))
|
|
1264
|
+
throw n === "stopped" ? new b(
|
|
1224
1265
|
"session.start() called from stopped state without a media source — call setMediaSource first"
|
|
1225
|
-
) : new
|
|
1226
|
-
|
|
1266
|
+
) : new b("session.start(): media source not bound");
|
|
1267
|
+
c || J(), T.resumeActive(), e.set({ lifecycle: "running" }), await new Promise((v) => requestAnimationFrame(() => v()));
|
|
1227
1268
|
const l = e.getSnapshot().pipeline;
|
|
1228
|
-
for (const [v,
|
|
1229
|
-
await
|
|
1269
|
+
for (const [v, y] of l.entries())
|
|
1270
|
+
await _(v, y);
|
|
1230
1271
|
e.set({ ready: !0 }), s.log("XR session started.");
|
|
1231
|
-
},
|
|
1232
|
-
const { lifecycle:
|
|
1233
|
-
|
|
1234
|
-
},
|
|
1235
|
-
await
|
|
1272
|
+
}, ee = async () => {
|
|
1273
|
+
const { lifecycle: n } = e.getSnapshot();
|
|
1274
|
+
n === "stopped" || n === "idle" || (P(), T.suspendAll(), e.set({ lifecycle: "stopped", ready: !1 }), s.log("XR session stopped."));
|
|
1275
|
+
}, Me = async () => {
|
|
1276
|
+
await ee(), T.releaseAll(), L.clear();
|
|
1277
|
+
const n = e.getSnapshot().cameraTexture;
|
|
1278
|
+
n && n.dispose(), C.clear(), r = void 0, k += 1, d = void 0, m = void 0, p = !1, e.set({
|
|
1236
1279
|
lifecycle: "idle",
|
|
1237
1280
|
cameraTexture: void 0,
|
|
1238
1281
|
source: void 0
|
|
1239
1282
|
}), s.log("XR session reset.");
|
|
1240
|
-
},
|
|
1241
|
-
const
|
|
1283
|
+
}, Re = async (n) => {
|
|
1284
|
+
const c = "legacy:registerXRPipeline", l = [];
|
|
1242
1285
|
await Promise.all(
|
|
1243
|
-
|
|
1244
|
-
const { model:
|
|
1245
|
-
if (!
|
|
1246
|
-
throw new
|
|
1247
|
-
|
|
1248
|
-
const
|
|
1249
|
-
await
|
|
1286
|
+
n.map(async (v) => {
|
|
1287
|
+
const { model: y } = v;
|
|
1288
|
+
if (!re[y])
|
|
1289
|
+
throw new b(`Invalid model: ${y}`);
|
|
1290
|
+
L.has(y) || l.push(y);
|
|
1291
|
+
const A = "options" in v ? v.options : void 0;
|
|
1292
|
+
await T.acquire(y, c, A);
|
|
1250
1293
|
})
|
|
1251
|
-
), l.length > 0 && (l.forEach((v) =>
|
|
1252
|
-
},
|
|
1253
|
-
const
|
|
1254
|
-
if (!
|
|
1255
|
-
throw new
|
|
1256
|
-
return
|
|
1257
|
-
},
|
|
1258
|
-
|
|
1294
|
+
), l.length > 0 && (l.forEach((v) => L.add(v)), s.log("XR pipeline registered."));
|
|
1295
|
+
}, Te = (n) => {
|
|
1296
|
+
const c = T.get(n);
|
|
1297
|
+
if (!c)
|
|
1298
|
+
throw new b(`Model ${n} is not registered in the pipeline`);
|
|
1299
|
+
return c.getModelNode();
|
|
1300
|
+
}, te = async (n, c) => {
|
|
1301
|
+
n <= 0 || c <= 0 || (e.set({ renderWidth: n, renderHeight: c }), d && (d.width !== n || d.height !== c) && (d.width = n, d.height = c, U(d), p && G()), s.log(`Source canvas set. Render size: ${n}x${c}`));
|
|
1302
|
+
}, xe = async (n) => {
|
|
1303
|
+
var y;
|
|
1304
|
+
S = n;
|
|
1305
|
+
const c = (y = n.getBoundingClientRect) == null ? void 0 : y.call(n), l = c && c.width > 0 ? Math.round(c.width) : n.clientWidth, v = c && c.height > 0 ? Math.round(c.height) : n.clientHeight;
|
|
1306
|
+
await te(l, v);
|
|
1307
|
+
}, Fe = (n) => {
|
|
1308
|
+
R = n.renderer, M = n.scene, x = n.camera, g = n.renderFrame;
|
|
1309
|
+
}, se = () => {
|
|
1310
|
+
if (!R || !M || !x)
|
|
1311
|
+
throw new b(
|
|
1312
|
+
"XR frame render requested before the R3F state was bound — make sure <XRScene> is mounted"
|
|
1313
|
+
);
|
|
1314
|
+
g ? g() : R.render(M, x);
|
|
1259
1315
|
};
|
|
1260
1316
|
return {
|
|
1261
1317
|
store: e,
|
|
1262
|
-
registry:
|
|
1263
|
-
setMediaSource:
|
|
1264
|
-
start:
|
|
1265
|
-
stop:
|
|
1266
|
-
reset:
|
|
1267
|
-
setFrameSource:
|
|
1268
|
-
registerXRPipeline:
|
|
1269
|
-
getModelNode:
|
|
1318
|
+
registry: T,
|
|
1319
|
+
setMediaSource: Se,
|
|
1320
|
+
start: ye,
|
|
1321
|
+
stop: ee,
|
|
1322
|
+
reset: Me,
|
|
1323
|
+
setFrameSource: ke,
|
|
1324
|
+
registerXRPipeline: Re,
|
|
1325
|
+
getModelNode: Te,
|
|
1270
1326
|
captureFrame: () => {
|
|
1271
|
-
if (!
|
|
1272
|
-
throw new
|
|
1327
|
+
if (!S)
|
|
1328
|
+
throw new b(
|
|
1273
1329
|
"session.captureFrame() called before the XRScene render target was bound — make sure <XRScene> is mounted"
|
|
1274
1330
|
);
|
|
1275
|
-
if (!
|
|
1276
|
-
throw new
|
|
1331
|
+
if (!R || !M || !x)
|
|
1332
|
+
throw new b(
|
|
1277
1333
|
"session.captureFrame() called before the R3F state was bound — make sure <XRScene> is mounted"
|
|
1278
1334
|
);
|
|
1279
|
-
return
|
|
1280
|
-
},
|
|
1281
|
-
_setRenderTarget: async (a) => {
|
|
1282
|
-
var S;
|
|
1283
|
-
g = a;
|
|
1284
|
-
const o = (S = a.getBoundingClientRect) == null ? void 0 : S.call(a), l = o && o.width > 0 ? Math.round(o.width) : a.clientWidth, v = o && o.height > 0 ? Math.round(o.height) : a.clientHeight;
|
|
1285
|
-
await Q(l, v);
|
|
1286
|
-
},
|
|
1287
|
-
_setRenderSize: Q,
|
|
1288
|
-
_setR3FState: (a) => {
|
|
1289
|
-
f = a.renderer, w = a.scene, p = a.camera;
|
|
1335
|
+
return se(), S.toDataURL("image/png");
|
|
1290
1336
|
},
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1337
|
+
getInputFrame: () => d ?? null,
|
|
1338
|
+
_setRenderTarget: xe,
|
|
1339
|
+
_setRenderSize: te,
|
|
1340
|
+
_setR3FState: Fe,
|
|
1341
|
+
_renderFrame: se,
|
|
1342
|
+
_getVideoElement: () => o,
|
|
1343
|
+
_setPerf: (n) => {
|
|
1344
|
+
if (t = n, e.getSnapshot().lifecycle !== "running" || !d) return;
|
|
1345
|
+
const c = e.getSnapshot().pipeline;
|
|
1346
|
+
for (const [l, v] of c.entries())
|
|
1296
1347
|
v.run({
|
|
1297
|
-
sourceCanvas:
|
|
1348
|
+
sourceCanvas: d,
|
|
1298
1349
|
renderWidth: e.getSnapshot().renderWidth,
|
|
1299
1350
|
renderHeight: e.getSnapshot().renderHeight,
|
|
1300
1351
|
getSource: () => e.getSnapshot().sourceKind,
|
|
1301
|
-
targetFps:
|
|
1352
|
+
targetFps: a(l)
|
|
1302
1353
|
});
|
|
1303
1354
|
}
|
|
1304
1355
|
};
|
|
1305
|
-
},
|
|
1306
|
-
const s =
|
|
1307
|
-
return
|
|
1308
|
-
|
|
1309
|
-
}, [
|
|
1310
|
-
},
|
|
1311
|
-
|
|
1356
|
+
}, Ct = (i) => {
|
|
1357
|
+
const s = oe(i), e = $(() => wt(s.current), []), t = JSON.stringify(i ?? {});
|
|
1358
|
+
return Ee(() => {
|
|
1359
|
+
e._setPerf(i);
|
|
1360
|
+
}, [e, t]), e;
|
|
1361
|
+
}, Q = /* @__PURE__ */ new Map(), St = () => ({
|
|
1362
|
+
register: (i, s) => {
|
|
1363
|
+
s ? Q.set(i, s) : Q.delete(i);
|
|
1364
|
+
},
|
|
1365
|
+
resolve: (i) => i ? Q.get(i) ?? null : null
|
|
1366
|
+
}), le = St(), ge = he(null), kt = ({ children: i }) => {
|
|
1367
|
+
const s = oe(/* @__PURE__ */ new Map()), e = ae((r, o) => {
|
|
1368
|
+
o ? s.current.set(r, o) : s.current.delete(r);
|
|
1369
|
+
}, []), t = ae(
|
|
1370
|
+
(r) => r ? s.current.get(r) ?? null : null,
|
|
1371
|
+
[]
|
|
1372
|
+
), a = $(() => ({ register: e, resolve: t }), [e, t]);
|
|
1373
|
+
return Ie(ge.Provider, { value: a }, i);
|
|
1374
|
+
}, me = () => ce(ge) ?? le, Dt = () => me().register, _t = () => me().resolve;
|
|
1375
|
+
le.register;
|
|
1376
|
+
const fe = he(void 0), Pt = () => {
|
|
1377
|
+
const i = ce(fe);
|
|
1312
1378
|
if (!i)
|
|
1313
|
-
throw new
|
|
1379
|
+
throw new b(
|
|
1314
1380
|
"XR context can only be used inside XRProvider, did you forget to wrap your app in <XRProvider>?"
|
|
1315
1381
|
);
|
|
1316
1382
|
return i;
|
|
1317
|
-
},
|
|
1318
|
-
const e =
|
|
1319
|
-
return /* @__PURE__ */
|
|
1383
|
+
}, Ot = ({ children: i, perf: s }) => {
|
|
1384
|
+
const e = Ct(s), t = $(() => ({ session: e }), [e]);
|
|
1385
|
+
return /* @__PURE__ */ ne(fe.Provider, { value: t, children: /* @__PURE__ */ ne(kt, { children: i }) });
|
|
1320
1386
|
};
|
|
1321
1387
|
export {
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1388
|
+
z as X,
|
|
1389
|
+
H as a,
|
|
1390
|
+
It as b,
|
|
1391
|
+
F as c,
|
|
1392
|
+
Ot as d,
|
|
1393
|
+
_t as e,
|
|
1394
|
+
He as f,
|
|
1395
|
+
Dt as g,
|
|
1396
|
+
I as l,
|
|
1397
|
+
Pt as u
|
|
1331
1398
|
};
|