@vincentt-xr/sdk 2.0.0-alpha.0 → 2.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (362) hide show
  1. package/GROUNDING.md +1813 -27
  2. package/dist/{DebuggerUI-CEfdddkZ.cjs → DebuggerUI-B45_Gb9d.cjs} +1 -1
  3. package/dist/{DebuggerUI-Dgop6Q6o.js → DebuggerUI-Cq28c18s.js} +1 -1
  4. package/dist/FaceMeshRenderer-6t8OsQhK.js +1195 -0
  5. package/dist/FaceMeshRenderer-lQLhYIW-.cjs +155 -0
  6. package/dist/FaceRetouch-CDrL0pXg.js +551 -0
  7. package/dist/FaceRetouch-DtNl8-4e.cjs +134 -0
  8. package/dist/audio/AudioEngine.d.ts +3 -1
  9. package/dist/audio/AudioPlayer.d.ts +29 -0
  10. package/dist/audio/AudioPlayerLogic.d.ts +4 -0
  11. package/dist/audio/index.d.ts +2 -0
  12. package/dist/components/Axes/index.d.ts +6 -0
  13. package/dist/components/FaceMesh/FaceMeshRenderer.d.ts +17 -0
  14. package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +22 -1
  15. package/dist/components/FaceMesh/canonical/FaceMeshMaterial.d.ts +25 -0
  16. package/dist/components/FaceMesh/canonical/assets.d.ts +3 -0
  17. package/dist/components/FaceMesh/canonical/geometryGroups.d.ts +9 -0
  18. package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +2 -0
  19. package/dist/components/FaceMesh/canonical/shaders.d.ts +2 -0
  20. package/dist/components/FaceMesh/deform.d.ts +3 -0
  21. package/dist/components/FaceMesh/index.d.ts +5 -0
  22. package/dist/components/FaceMesh/material/defaults.d.ts +74 -0
  23. package/dist/components/FaceMesh/material/index.d.ts +1 -0
  24. package/dist/components/FaceMesh/material/options.d.ts +5 -1
  25. package/dist/components/FaceMesh/material/support.d.ts +7 -2
  26. package/dist/components/FaceMesh/material/textures.d.ts +4 -0
  27. package/dist/components/FaceMesh/material/types.d.ts +4 -2
  28. package/dist/components/FaceRetouch/Background.d.ts +9 -5
  29. package/dist/components/FaceRetouch/FaceRetouch.d.ts +47 -0
  30. package/dist/components/FaceRetouch/index.d.ts +2 -6
  31. package/dist/components/FaceRetouch/landmarkMask.d.ts +4 -6
  32. package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +4 -7
  33. package/dist/components/FaceTracker/index.d.ts +5 -3
  34. package/dist/components/FaceWarp3D/FaceWarp3DHead.d.ts +15 -0
  35. package/dist/components/FaceWarp3D/defaults.d.ts +37 -0
  36. package/dist/components/FaceWarp3D/index.d.ts +3 -0
  37. package/dist/components/FaceWarp3D/warp.d.ts +13 -0
  38. package/dist/components/HandJoints2D/index.d.ts +6 -0
  39. package/dist/components/HandTracker/index.d.ts +2 -2
  40. package/dist/components/HandTracking2D/HandVisual.d.ts +12 -0
  41. package/dist/components/HandTracking2D/index.d.ts +5 -0
  42. package/dist/components/HandTracking2DController/index.d.ts +5 -0
  43. package/dist/components/HandTracking3D/index.d.ts +6 -0
  44. package/dist/components/LightObject/index.d.ts +20 -0
  45. package/dist/components/LookAt/index.d.ts +23 -0
  46. package/dist/components/MeshRenderer/index.d.ts +58 -0
  47. package/dist/components/PerspectiveCamera/index.d.ts +2 -8
  48. package/dist/components/PinToMesh/index.d.ts +47 -0
  49. package/dist/components/RenderGroupLayer/index.d.ts +6 -0
  50. package/dist/components/SceneObjectRenderer/index.d.ts +16 -0
  51. package/dist/components/Screen2D/ScreenTransformFrame.d.ts +11 -0
  52. package/dist/components/Screen2D/ScreenTransformHelpers.d.ts +19 -0
  53. package/dist/components/Screen2D/ScreenTransformInteraction.d.ts +18 -0
  54. package/dist/components/Screen2D/index.d.ts +5 -0
  55. package/dist/components/ScreenGlass/defaults.d.ts +3 -0
  56. package/dist/components/ScreenGlass/index.d.ts +5 -0
  57. package/dist/components/ScreenGlass/shaders.d.ts +2 -0
  58. package/dist/components/ScreenGlass/types.d.ts +10 -0
  59. package/dist/components/ScreenGradient/defaults.d.ts +5 -0
  60. package/dist/components/ScreenGradient/index.d.ts +5 -0
  61. package/dist/components/ScreenGradient/shaders.d.ts +2 -0
  62. package/dist/components/ScreenGradient/types.d.ts +19 -0
  63. package/dist/components/ScreenImage/ScreenImage.d.ts +4 -0
  64. package/dist/components/ScreenImage/components/DynamicTextureSync.d.ts +9 -0
  65. package/dist/components/ScreenImage/components/ScreenImageBody.d.ts +9 -0
  66. package/dist/components/ScreenImage/components/ScreenImageFrame.d.ts +9 -0
  67. package/dist/components/ScreenImage/components/ScreenImageTransformFrame.d.ts +9 -0
  68. package/dist/components/ScreenImage/fit.d.ts +52 -0
  69. package/dist/components/ScreenImage/index.d.ts +8 -0
  70. package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts +9 -0
  71. package/dist/components/ScreenImage/materials/ScreenImageMatcap.d.ts +3 -0
  72. package/dist/components/ScreenImage/source.d.ts +16 -0
  73. package/dist/components/ScreenImage/types.d.ts +37 -0
  74. package/dist/components/ScreenImage/utils/ScreenImageConstants.d.ts +9 -0
  75. package/dist/components/ScreenImage/utils/ScreenImageHelpers.d.ts +4 -0
  76. package/dist/components/ScreenShape/geometry.d.ts +5 -0
  77. package/dist/components/ScreenShape/index.d.ts +5 -0
  78. package/dist/components/ScreenShape/types.d.ts +41 -0
  79. package/dist/components/ScreenSpaceUI/ScreenTransformGuide.d.ts +15 -0
  80. package/dist/components/ScreenSpaceUI/ScreenTransformOverlayGuide.d.ts +9 -0
  81. package/dist/components/ScreenSpaceUI/index.d.ts +2 -0
  82. package/dist/components/ScreenText/components/ScreenTextBody.d.ts +2 -0
  83. package/dist/components/ScreenText/components/ScreenTextTransformFrame.d.ts +9 -0
  84. package/dist/components/ScreenText/constants.d.ts +5 -0
  85. package/dist/components/ScreenText/index.d.ts +6 -0
  86. package/dist/components/ScreenText/types.d.ts +31 -0
  87. package/dist/components/ScreenText/utils/ScreenTextHelpers.d.ts +14 -0
  88. package/dist/components/ScreenText/utils/ScreenTextMappings.d.ts +17 -0
  89. package/dist/components/SequentialRenderPasses/index.d.ts +6 -0
  90. package/dist/components/TextLabel/TextLabel.d.ts +5 -0
  91. package/dist/components/TextLabel/alignment.d.ts +4 -0
  92. package/dist/components/TextLabel/bend.d.ts +9 -0
  93. package/dist/components/TextLabel/canvas/buildTextLabelTexture.d.ts +4 -0
  94. package/dist/components/TextLabel/canvas/canvasTypography.d.ts +13 -0
  95. package/dist/components/TextLabel/canvas/drawBackground.d.ts +21 -0
  96. package/dist/components/TextLabel/canvas/drawEffects.d.ts +34 -0
  97. package/dist/components/TextLabel/canvas/drawText.d.ts +19 -0
  98. package/dist/components/TextLabel/components/TextLabelMesh.d.ts +13 -0
  99. package/dist/components/TextLabel/constants.d.ts +2 -0
  100. package/dist/components/TextLabel/hooks/useTextLabelTexture.d.ts +7 -0
  101. package/dist/components/TextLabel/index.d.ts +2 -62
  102. package/dist/components/TextLabel/layout/measurement.d.ts +8 -0
  103. package/dist/components/TextLabel/layout/resolveTextLayout.d.ts +25 -0
  104. package/dist/components/TextLabel/layout/wrapText.d.ts +6 -0
  105. package/dist/components/TextLabel/types.d.ts +227 -0
  106. package/dist/components/TrackingAnchor/index.d.ts +18 -2
  107. package/dist/components/Transform3D/index.d.ts +30 -0
  108. package/dist/components/XRMediaSurface/index.d.ts +6 -6
  109. package/dist/components/XRScene/index.d.ts +14 -3
  110. package/dist/components/XRViewport/XRViewport.d.ts +11 -0
  111. package/dist/components/XRViewport/components/HiddenMedia.d.ts +7 -0
  112. package/dist/components/XRViewport/components/MediaAssetPicker.d.ts +8 -0
  113. package/dist/components/XRViewport/components/MediaControlStack.d.ts +13 -0
  114. package/dist/components/XRViewport/components/XRViewportLayers.d.ts +30 -0
  115. package/dist/components/XRViewport/constants.d.ts +24 -0
  116. package/dist/components/XRViewport/contexts.d.ts +6 -0
  117. package/dist/components/XRViewport/hooks/useViewportExternalFaces.d.ts +13 -0
  118. package/dist/components/XRViewport/hooks/useViewportMediaSource.d.ts +16 -0
  119. package/dist/components/XRViewport/hooks/useViewportMediaTexture.d.ts +10 -0
  120. package/dist/components/XRViewport/hooks/useViewportPipeline.d.ts +41 -0
  121. package/dist/components/XRViewport/hooks/useViewportRenderState.d.ts +47 -0
  122. package/dist/components/XRViewport/index.d.ts +3 -101
  123. package/dist/components/XRViewport/layout.d.ts +16 -0
  124. package/dist/components/XRViewport/slots.d.ts +16 -0
  125. package/dist/components/XRViewport/types.d.ts +89 -0
  126. package/dist/components/XRViewport/useViewportSize.d.ts +3 -0
  127. package/dist/components/index.d.ts +19 -0
  128. package/dist/contexts-BKvTIayr.cjs +1 -0
  129. package/dist/contexts-Bk5zz_VR.js +214 -0
  130. package/dist/debug-ui/component-editor/index.d.ts +19 -0
  131. package/dist/debug-ui/default-setting/index.d.ts +37 -0
  132. package/dist/debug-ui/default-setting.cjs +21 -0
  133. package/dist/debug-ui/default-setting.js +777 -0
  134. package/dist/debug-ui/face-mesh-renderer-settings/index.d.ts +8 -0
  135. package/dist/debug-ui/face-retouch/index.d.ts +11 -0
  136. package/dist/debug-ui/face-warp-3d/index.d.ts +6 -0
  137. package/dist/debug-ui/hand-joints-2d/index.d.ts +6 -0
  138. package/dist/debug-ui/hand-tracking-2d/index.d.ts +6 -0
  139. package/dist/debug-ui/hand-tracking-2d-controller/index.d.ts +6 -0
  140. package/dist/debug-ui/hand-tracking-3d/index.d.ts +6 -0
  141. package/dist/debug-ui/hierarchy-panel/index.d.ts +29 -0
  142. package/dist/debug-ui/index.d.ts +46 -0
  143. package/dist/debug-ui/layer-panel/index.d.ts +18 -0
  144. package/dist/debug-ui/media-source/MediaSourceSwitcher.d.ts +10 -0
  145. package/dist/debug-ui/media-source/index.d.ts +6 -0
  146. package/dist/debug-ui/media-source/media-preset.d.ts +9 -0
  147. package/dist/debug-ui/media-source/session.d.ts +12 -0
  148. package/dist/debug-ui/media-source.cjs +1 -0
  149. package/dist/debug-ui/media-source.js +9 -0
  150. package/dist/debug-ui/mesh-renderer-settings/index.d.ts +6 -0
  151. package/dist/debug-ui/object-inspector-panel/index.d.ts +25 -0
  152. package/dist/debug-ui/pin-to-mesh-settings/index.d.ts +10 -0
  153. package/dist/debug-ui/scene-editor/context.d.ts +38 -0
  154. package/dist/debug-ui/scene-editor/hierarchy-panel.d.ts +9 -0
  155. package/dist/debug-ui/scene-editor/index.d.ts +11 -0
  156. package/dist/debug-ui/scene-editor/inspector-panel.d.ts +7 -0
  157. package/dist/debug-ui/scene-editor/store-utils.d.ts +2 -0
  158. package/dist/debug-ui/scene-editor/viewport-toolbar.d.ts +4 -0
  159. package/dist/debug-ui/scene-editor/viewport.d.ts +21 -0
  160. package/dist/debug-ui/screen-glass-settings/index.d.ts +6 -0
  161. package/dist/debug-ui/screen-gradient-settings/GradientStopsEditor.d.ts +5 -0
  162. package/dist/debug-ui/screen-gradient-settings/index.d.ts +6 -0
  163. package/dist/debug-ui/screen-gradient-settings/styles.d.ts +3 -0
  164. package/dist/debug-ui/screen-image-settings/index.d.ts +9 -0
  165. package/dist/debug-ui/screen-shape-settings/index.d.ts +6 -0
  166. package/dist/debug-ui/screen-text-settings/NumberField.d.ts +9 -0
  167. package/dist/debug-ui/screen-text-settings/ScreenTextSettingsPanel.d.ts +3 -0
  168. package/dist/debug-ui/screen-text-settings/controls/NestedSection.d.ts +7 -0
  169. package/dist/debug-ui/screen-text-settings/controls/Section.d.ts +8 -0
  170. package/dist/debug-ui/screen-text-settings/controls/icons.d.ts +4 -0
  171. package/dist/debug-ui/screen-text-settings/index.d.ts +2 -0
  172. package/dist/debug-ui/screen-text-settings/sections/BackgroundSection.d.ts +8 -0
  173. package/dist/debug-ui/screen-text-settings/sections/ContentSection.d.ts +5 -0
  174. package/dist/debug-ui/screen-text-settings/sections/EffectsSection.d.ts +25 -0
  175. package/dist/debug-ui/screen-text-settings/sections/StyleSection.d.ts +5 -0
  176. package/dist/debug-ui/screen-text-settings/sections/TransformSection.d.ts +5 -0
  177. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundFillControls.d.ts +2 -0
  178. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundSizeControls.d.ts +2 -0
  179. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundStrokeControls.d.ts +2 -0
  180. package/dist/debug-ui/screen-text-settings/sections/background/types.d.ts +7 -0
  181. package/dist/debug-ui/screen-text-settings/sections/effects/BendEffectSection.d.ts +4 -0
  182. package/dist/debug-ui/screen-text-settings/sections/effects/EffectChrome.d.ts +21 -0
  183. package/dist/debug-ui/screen-text-settings/sections/effects/GlowEffectSection.d.ts +4 -0
  184. package/dist/debug-ui/screen-text-settings/sections/effects/InnerShadowEffectSection.d.ts +4 -0
  185. package/dist/debug-ui/screen-text-settings/sections/effects/OutlineEffectSection.d.ts +4 -0
  186. package/dist/debug-ui/screen-text-settings/sections/effects/ShadowEffectSection.d.ts +4 -0
  187. package/dist/debug-ui/screen-text-settings/sections/effects/types.d.ts +14 -0
  188. package/dist/debug-ui/screen-text-settings/settings.d.ts +90 -0
  189. package/dist/debug-ui/screen-text-settings/styles.d.ts +26 -0
  190. package/dist/debug-ui/screen-text-settings/transform.d.ts +6 -0
  191. package/dist/debug-ui/screen-text-settings/types.d.ts +9 -0
  192. package/dist/debug-ui/settings-panel/index.d.ts +38 -0
  193. package/dist/debug-ui/shared/MaterialSection.d.ts +13 -0
  194. package/dist/debug-ui/shared/ScreenTransformSection.d.ts +32 -0
  195. package/dist/debug-ui/transform-3d-settings/index.d.ts +11 -0
  196. package/dist/debug-ui.cjs +10 -0
  197. package/dist/debug-ui.js +8861 -0
  198. package/dist/drawNormalizedCrop-B-7zU-Xw.cjs +1 -0
  199. package/dist/drawNormalizedCrop-CRdIDyl1.js +44 -0
  200. package/dist/entries/all.d.ts +1 -0
  201. package/dist/entries/core.d.ts +55 -4
  202. package/dist/entries/face-effects.d.ts +12 -8
  203. package/dist/entries/internal.d.ts +1 -1
  204. package/dist/entries/low-level.d.ts +21 -4
  205. package/dist/entries/scene-object.d.ts +10 -0
  206. package/dist/entries/tracking.d.ts +25 -2
  207. package/dist/face-action/index.d.ts +34 -0
  208. package/dist/face-attribute-expression/index.d.ts +10 -0
  209. package/dist/face-effects.cjs +1 -252
  210. package/dist/face-effects.js +1404 -2613
  211. package/dist/face-info/blendshapes.d.ts +3 -0
  212. package/dist/face-info/derive.d.ts +8 -0
  213. package/dist/face-info/drawFaceCrop.d.ts +10 -0
  214. package/dist/face-info/getFaceInfo.d.ts +8 -0
  215. package/dist/face-info/index.d.ts +8 -0
  216. package/dist/face-info/landmarks.d.ts +21 -0
  217. package/dist/face-info/types.d.ts +52 -0
  218. package/dist/getFaceInfo-CyPAKh3h.cjs +1 -0
  219. package/dist/getFaceInfo-hUKMLFcr.js +118 -0
  220. package/dist/hand/handJoints2d.d.ts +49 -0
  221. package/dist/hand/handTracking2d.d.ts +33 -0
  222. package/dist/hand/handTracking2dController.d.ts +65 -0
  223. package/dist/hand/handTracking3d.d.ts +52 -0
  224. package/dist/hooks/faceResults.d.ts +1 -1
  225. package/dist/hooks/useCalibrationFaceTracker.d.ts +1 -1
  226. package/dist/hooks/useFaceAction.d.ts +3 -0
  227. package/dist/hooks/useFaceAttributeExpression.d.ts +16 -0
  228. package/dist/hooks/useFaceDetection.d.ts +18 -0
  229. package/dist/hooks/useFaceInfo.d.ts +7 -0
  230. package/dist/hooks/useFaceInfos.d.ts +15 -0
  231. package/dist/hooks/useFaceResultReader.d.ts +10 -0
  232. package/dist/hooks/useFrameCapture.d.ts +18 -0
  233. package/dist/hooks/useMediaRecorder.d.ts +14 -2
  234. package/dist/hooks/useOwnedTexture.d.ts +6 -0
  235. package/dist/hooks/useTrackerContext.d.ts +2 -2
  236. package/dist/hooks/useXRSession/facade.d.ts +3 -0
  237. package/dist/hooks/useXRSession/registry.d.ts +9 -0
  238. package/dist/index-82aENaRq.cjs +1 -0
  239. package/dist/{index-BD-JWMjE.js → index-BgQAX8rs.js} +198 -134
  240. package/dist/index-CEc1CAmB.cjs +1 -0
  241. package/dist/index-CTHeBQOs.js +178 -0
  242. package/dist/index-D4bLzBr6.cjs +1 -0
  243. package/dist/index-DSpzjuqd.cjs +1 -0
  244. package/dist/index-Di70KWuY.js +2049 -0
  245. package/dist/index-DzEzkBIH.cjs +17 -0
  246. package/dist/{index-CYMj26Ej.js → index-JygePOwo.js} +565 -498
  247. package/dist/index-UC0Ux38E.js +111 -0
  248. package/dist/{internal-KOKJDxCG.js → internal-DuMPpcbp.js} +86 -80
  249. package/dist/internal-i2JkM85N.cjs +1 -0
  250. package/dist/internal.cjs +1 -1
  251. package/dist/internal.js +2 -2
  252. package/dist/low-level.cjs +1 -1
  253. package/dist/low-level.js +959 -632
  254. package/dist/main.cjs +1 -1
  255. package/dist/main.js +556 -455
  256. package/dist/material/MaterialFill.d.ts +20 -0
  257. package/dist/material/MeshMaterial.d.ts +14 -0
  258. package/dist/material/index.d.ts +2 -0
  259. package/dist/models/Body/index.d.ts +1 -1
  260. package/dist/models/Face/index.d.ts +1 -0
  261. package/dist/models/Face/rotation.d.ts +3 -0
  262. package/dist/models/Hand/HandTracking.d.ts +2 -0
  263. package/dist/render/layerStack.d.ts +29 -0
  264. package/dist/render/renderGroups.d.ts +47 -0
  265. package/dist/render/renderPasses.d.ts +18 -0
  266. package/dist/renderGroups-C9QJrAXu.js +152 -0
  267. package/dist/renderGroups-Dvm1J4lW.cjs +1 -0
  268. package/dist/renderLayers-lDtL0_43.cjs +1 -0
  269. package/dist/renderLayers-zrK2pv5O.js +13 -0
  270. package/dist/renderPasses-CT5iBwtg.cjs +21 -0
  271. package/dist/renderPasses-CjgG_6kQ.js +1500 -0
  272. package/dist/scene-object/index.d.ts +146 -0
  273. package/dist/scene-object/runtimeRegistry.d.ts +14 -0
  274. package/dist/scene-object.cjs +1 -0
  275. package/dist/scene-object.js +40 -0
  276. package/dist/screen-image/screenImagePresets.d.ts +13 -0
  277. package/dist/screen-image/screenImageSettings.d.ts +57 -0
  278. package/dist/screen-text/screenTextPresets.d.ts +6 -0
  279. package/dist/screen-text/screenTextSettings.d.ts +147 -0
  280. package/dist/screen2d/coordinates.d.ts +87 -0
  281. package/dist/screen2d/index.d.ts +7 -0
  282. package/dist/screen2d/material-runtime.d.ts +32 -0
  283. package/dist/screen2d/screenTransform.d.ts +47 -0
  284. package/dist/screen2d/smoothing.d.ts +5 -0
  285. package/dist/screen2d/texture.d.ts +10 -0
  286. package/dist/screen2d/transform.d.ts +25 -0
  287. package/dist/screen2d/types.d.ts +90 -0
  288. package/dist/{selectors-Bub0o3qy.cjs → selectors-2QvO5RYY.cjs} +1 -1
  289. package/dist/{selectors-BHO6TnoC.js → selectors-Dw3C-gSk.js} +5 -5
  290. package/dist/services/MetricsCollector/index.d.ts +7 -1
  291. package/dist/services/MetricsCollector/utils.d.ts +1 -1
  292. package/dist/session-BChR4TgK.cjs +31 -0
  293. package/dist/session-C1StwN0f.js +363 -0
  294. package/dist/texture/animatedTexture.d.ts +61 -0
  295. package/dist/texture/animationMath.d.ts +8 -0
  296. package/dist/texture/sourceResolver.d.ts +30 -0
  297. package/dist/tracking.cjs +1 -1
  298. package/dist/tracking.js +582 -298
  299. package/dist/types/face.d.ts +1 -13
  300. package/dist/types/hand-tracking.d.ts +8 -0
  301. package/dist/types/index.d.ts +0 -1
  302. package/dist/types/nodes.d.ts +4 -1
  303. package/dist/types/session.d.ts +2 -0
  304. package/dist/types/tracking.d.ts +8 -2
  305. package/dist/types/util-types.d.ts +1 -1
  306. package/dist/useFaceInfo-C-WfMlYp.cjs +1 -0
  307. package/dist/useFaceInfo-CwsHZedC.js +62 -0
  308. package/dist/useFaceResults-CkFc1egZ.js +19 -0
  309. package/dist/useFaceResults-VcFdOUFV.cjs +1 -0
  310. package/dist/utils/drawNormalizedCrop.d.ts +22 -0
  311. package/dist/utils/integrity.d.ts +4 -25
  312. package/dist/utils/landmarkBounds.d.ts +1 -1
  313. package/dist/utils/logger.d.ts +2 -2
  314. package/dist/x-Co8jPhP0.js +127 -0
  315. package/dist/x-KC8E4Gaj.cjs +41 -0
  316. package/docs/MIGRATION_GUIDE.md +589 -0
  317. package/docs/SDK_DOCUMENTATION.md +1155 -0
  318. package/docs/api/audio.mdx +36 -0
  319. package/docs/api/behavior-components.mdx +164 -0
  320. package/docs/api/capture-recording.mdx +103 -0
  321. package/docs/api/debug-ui-scene-editor.mdx +249 -0
  322. package/docs/api/face-actions.mdx +104 -0
  323. package/docs/api/face-attribute-expression.mdx +87 -0
  324. package/docs/api/face-binding-anchors.mdx +53 -0
  325. package/docs/api/face-effects.mdx +19 -0
  326. package/docs/api/face-info.mdx +112 -0
  327. package/docs/api/face-mesh.mdx +98 -0
  328. package/docs/api/face-painting.mdx +33 -0
  329. package/docs/api/face-retouching.mdx +73 -0
  330. package/docs/api/material-render-state.mdx +132 -0
  331. package/docs/api/runtime-3d.mdx +311 -0
  332. package/docs/api/scene-object-store.mdx +213 -0
  333. package/docs/api/screen-image.mdx +41 -0
  334. package/docs/api/screen-shape.mdx +64 -0
  335. package/docs/api/screen-space-coordinates.mdx +41 -0
  336. package/docs/api/screen-text.mdx +39 -0
  337. package/docs/api/texture-sources.mdx +55 -0
  338. package/docs/api/trackers.mdx +46 -0
  339. package/docs/docs.json +70 -0
  340. package/docs/examples/scene-object-recipes.mdx +194 -0
  341. package/docs/guides/harness-debug-ui.mdx +139 -0
  342. package/docs/inspector/hierarchy-panel.mdx +51 -0
  343. package/docs/inspector/object-inspector-panel.mdx +51 -0
  344. package/docs/introduction.mdx +121 -0
  345. package/docs/scene-objects/overview.mdx +127 -0
  346. package/docs/scene-objects/render-groups.mdx +77 -0
  347. package/docs/scene-objects/runtime-rendering.mdx +75 -0
  348. package/docs/texture-api-consumer.md +58 -0
  349. package/package.json +55 -7
  350. package/dist/components/FaceRetouch/retouchRegions.d.ts +0 -23
  351. package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +0 -12
  352. package/dist/index-B5J4gzn2.cjs +0 -1
  353. package/dist/index-BVBVg96G.cjs +0 -1
  354. package/dist/index-DtwHElmG.js +0 -157
  355. package/dist/index-PtFmD6B8.cjs +0 -1
  356. package/dist/internal-DH5d1Xnq.cjs +0 -1
  357. package/dist/modules/XRClient/index.d.ts +0 -9
  358. package/dist/modules/index.d.ts +0 -1
  359. package/dist/services/ApiService.d.ts +0 -12
  360. package/dist/types/client.d.ts +0 -3
  361. package/dist/useFaceResults-cpsSmKKc.js +0 -81
  362. package/dist/useFaceResults-hVUJYSEY.cjs +0 -1
package/GROUNDING.md CHANGED
@@ -10,7 +10,8 @@ or spelunk `node_modules` for the API.
10
10
  The API is split into task-domain entry points. Import each symbol from the door
11
11
  this reference shows for it; the imports here are literal.
12
12
 
13
- - `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useMediaRecorder`, `dataURLtoFile`), audio.
13
+ - `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, runtime renderers (`Transform3D`, `MeshRenderer`, `SceneObjectRenderer`), `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useFrameCapture`, `useMediaRecorder`, `dataURLtoFile`), audio.
14
+ - `@vincentt-xr/sdk/scene-object` — **public authoring state**: scene-object/component types, factories, selectors, and immutable store operations.
14
15
  - `@vincentt-xr/sdk/tracking` — **trackers**: face/hand/gesture/body/segmentation, `TrackingAnchor`, `GestureTrigger`, the bare `FaceMesh`, `useFaceResults`.
15
16
  - `@vincentt-xr/sdk/face-effects` — **face deep**: FaceMesh material config, retouch, canonical mesh, head-binding.
16
17
  - `@vincentt-xr/sdk/low-level` — **escape hatch**: raw model-node reads, selector hooks, custom-tracker plumbing. Reach here only when core + tracking can't express it.
@@ -24,6 +25,50 @@ logic is R3F `useFrame`; per-mount setup is `useEffect`.
24
25
  > patterns) into the project's `GROUNDING.md`. Edit the SDK API here, in the repo
25
26
  > that owns the API.
26
27
 
28
+ ### Internal source layout
29
+
30
+ The public import doors above are stable. Internal files are organized by domain
31
+ so shared behavior stays reusable instead of duplicated in large components:
32
+
33
+ - `lib/components/ScreenImage/` — thin `ScreenImage.tsx` orchestrator plus
34
+ reusable body, frame, transform-frame, material, matcap, shader, source, and
35
+ helper modules.
36
+ - `lib/components/ScreenText/` — thin `ScreenText.tsx` orchestrator plus
37
+ reusable body, transform-frame, constants, mapping, and helper modules.
38
+ - `lib/components/TextLabel/` — low-level canvas text renderer split into
39
+ texture building, layout, style, hooks, and body modules.
40
+ - `lib/components/XRViewport/` — thin viewport orchestrator plus reusable
41
+ provider, stage, overlays, media, control, sizing, slot, and layout modules.
42
+ - `lib/components/Screen2D/` — shared screen-transform interaction/frame helpers
43
+ used by screen-space authoring components.
44
+ - `lib/debug-ui/screen-text-settings/` — panel orchestrator plus reusable
45
+ sections, controls, settings, styles, transform helpers, and types.
46
+ - `lib/render/` — render-group, render-pass, and layer-stack internals.
47
+ - `lib/hand/`, `lib/screen-image/`, and `lib/screen-text/` — domain-level
48
+ primitives, presets, settings, and tests that used to live flat in `lib/`.
49
+
50
+ When adding features, keep public exports at the documented import doors and put
51
+ implementation details in the matching domain folder. Do not reintroduce
52
+ component-local duplicate shader, transform, text-rendering, or settings logic.
53
+
54
+ ### Example asset layout
55
+
56
+ Example-app assets are organized by purpose, not by whichever page first needed
57
+ them:
58
+
59
+ - `example/src/assets/media/images/` — reusable still textures and sample images.
60
+ - `example/src/assets/media/images/people/` — face/body tracking sample stills.
61
+ - `example/src/assets/media/images/screenshots/` — screenshot reference images.
62
+ - `example/src/assets/media/videos/` — sample MP4/WebM sources.
63
+ - `example/src/assets/media/gifs/` — animated GIF source files.
64
+ - `example/src/assets/media/sequences/<demo>/` — ordered image-sequence frames.
65
+ - `example/src/assets/models/` — GLB files used by example 3D scenes.
66
+ - `example/src/assets/face-effects/` — domain-owned face-effect assets.
67
+
68
+ Do not flatten all example assets into the root. Do not add new `utils/` media
69
+ folders. Compression is a separate visual/behavior review because it can change
70
+ tracking fixtures and texture output.
71
+
27
72
  ---
28
73
 
29
74
  ## Screen-space layout (the primary tool)
@@ -54,7 +99,9 @@ Coordinates are normalized: -1 to +1, with +y up. The center of the screen is `(
54
99
  <mesh name="frame"><planeGeometry args={[1, 1]} /><meshBasicMaterial map={frameTex} transparent /></mesh>
55
100
  </ScreenTransform>
56
101
 
57
- // Bottom-center prompt
102
+ // Bottom-center prompt.
103
+ // Prefer <ScreenText> for ordinary screen copy; use this low-level pairing only
104
+ // when you need custom anchors.
58
105
  <ScreenTransform anchors={{ left: -0.4, right: 0.4, top: -0.5, bottom: -0.9 }}>
59
106
  <TextLabel name="prompt" text="Show your hand" fontSize={220} />
60
107
  </ScreenTransform>
@@ -84,6 +131,196 @@ const [sx, sy] = computeContainScale(rectAspect, 1200 / 800); // content w/h rat
84
131
  </mesh>;
85
132
  ```
86
133
 
134
+ ### `<ScreenImage>` — Effect House-style screen-space image object
135
+
136
+ Use this for screen-space image planes such as logos, stickers, frames, badges,
137
+ and photo placeholders. `ScreenImage` owns its `ScreenTransform`; for normal
138
+ authoring prefer `position`, `size`, and `rotation`. `anchors` still exist as
139
+ the lower-level layout escape hatch. If no source is provided, it renders the
140
+ SDK-owned square default image.
141
+
142
+ ### `<ScreenShape>` — editable vector primitives
143
+
144
+ Use `ScreenShape` for circles, ellipses, rectangles, rounded rectangles, regular
145
+ polygons, stars, and lines. Do not generate raster image assets for these basic
146
+ shapes. Placement and drag editing use the shared `ScreenTransform` settings.
147
+ Circles, regular polygons, and stars preserve equal axes inside non-square
148
+ transforms; ellipses and rectangles fill the transform.
149
+
150
+ ```tsx
151
+ import { ScreenShape, ScreenSpaceUI } from "@vincentt-xr/sdk";
152
+
153
+ <ScreenSpaceUI>
154
+ <ScreenShape
155
+ transform={{ position: { x: 0, y: 0 }, size: { width: 220, height: 220 } }}
156
+ settings={{
157
+ kind: "star",
158
+ fillColor: "#ffd54a",
159
+ strokeColor: "#ffffff",
160
+ strokeWidth: 4,
161
+ sides: 5,
162
+ innerRadiusRatio: 0.45,
163
+ }}
164
+ />
165
+ </ScreenSpaceUI>;
166
+ ```
167
+
168
+ Scene objects pair `ScreenTransform` with `ScreenShape`. Import scene-object
169
+ factories from `@vincentt-xr/sdk/scene-object` and the live
170
+ `ScreenShapeSettingsPanel` from `@vincentt-xr/sdk/debug-ui`.
171
+
172
+ Settings: `kind`; `fillMode`, `fillColor`, `fillOpacity`; `strokeColor`,
173
+ `strokeOpacity`, `strokeWidth`, `dashed`; `cornerRadius` for rounded rectangles;
174
+ `sides` for polygons/stars; `innerRadiusRatio` for stars; and normalized
175
+ `lineStart`/`lineEnd` `{ x, y }` coordinates for lines.
176
+
177
+ Set `fillMode: "gradient"` and provide `gradient: { type, stops, angle, center,
178
+ radius }`. Types are `linear`, `radial`, and `angular` (conic). A gradient accepts
179
+ 2–8 `{ offset, color }` stops. `ScreenGradientMaterial` is a separate reusable
180
+ component exported from core; its settings panel is exported from debug-ui.
181
+
182
+ Set `fillMode: "glass"` and provide `glass: { tint, opacity, frost, highlight }`
183
+ for lightweight translucent glass. `ScreenGlassMaterial` is a separate reusable
184
+ component. It is a stylized overlay material, not backdrop blur/refraction.
185
+
186
+ ### Shared material primitives
187
+
188
+ Import shared material primitives from `@vincentt-xr/sdk` only:
189
+
190
+ ```tsx
191
+ import {
192
+ DEFAULT_MATERIAL_FILL,
193
+ MaterialFill,
194
+ MeshMaterial,
195
+ createTwoStopGradient,
196
+ type MaterialFillSettings,
197
+ type MeshMaterialSettings,
198
+ } from "@vincentt-xr/sdk";
199
+ ```
200
+
201
+ Use `MaterialFill` when a component needs the same solid/gradient/texture fill
202
+ system as `ScreenShape` and FaceMesh masks. `MaterialFillSettings.source` is
203
+ `"solid" | "gradient" | "texture"`; pass `color`, `opacity`, optional
204
+ `textureUrl`, and optional `gradient`.
205
+
206
+ ```tsx
207
+ const fill: MaterialFillSettings = {
208
+ ...DEFAULT_MATERIAL_FILL,
209
+ source: "gradient",
210
+ color: "#38bdf8",
211
+ opacity: 0.8,
212
+ gradient: createTwoStopGradient("#38bdf8"),
213
+ };
214
+
215
+ <mesh>
216
+ <planeGeometry args={[1, 1]} />
217
+ <MaterialFill settings={fill} />
218
+ </mesh>;
219
+ ```
220
+
221
+ Use `MeshMaterial` for serialized mesh material settings shared by
222
+ `MeshRenderer`; do not re-create local `<meshBasicMaterial>` /
223
+ `<meshStandardMaterial>` branches in scene-object components.
224
+
225
+ ```tsx
226
+ import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
227
+
228
+ <ScreenSpaceUI>
229
+ <ScreenImage
230
+ name="defaultSquareImage"
231
+ anchors={{ left: -0.35, right: 0.35, top: 0.35, bottom: -0.35 }}
232
+ fit="contain"
233
+ />
234
+ </ScreenSpaceUI>
235
+
236
+ <ScreenSpaceUI>
237
+ <ScreenImage
238
+ name="logo"
239
+ source={{ kind: "image", src: "https://cdn.../logo.png" }}
240
+ coordinateSpace="canvas"
241
+ canvasSize={{ width: 720, height: 1280 }}
242
+ position={[-180, 464]}
243
+ size={[216, 160]}
244
+ stretchMode="fit"
245
+ opacity={0.95}
246
+ />
247
+ </ScreenSpaceUI>
248
+ ```
249
+
250
+ Props:
251
+
252
+ - `source`: `{ kind: "default" }`, `{ kind: "image", src }`, or `{ kind: "texture", texture }`.
253
+ - `src`: shorthand for `{ kind: "image", src }`.
254
+ - `fit`: `"stretch" | "contain" | "cover"`; default `"contain"`.
255
+ - `coordinateSpace`: `"normalized" | "canvas"`; default `"normalized"`.
256
+ - `canvasSize`: design canvas size for `coordinateSpace="canvas"`; default
257
+ `{ width: 720, height: 1280 }`.
258
+ - `position`, `size`, `rotation`: author-facing screen-object transform.
259
+ In normalized mode, `position` is the rect center in normalized screen space
260
+ and `size` is normalized rect width/height. In canvas mode, `position` and
261
+ `size` are design-canvas pixels with center origin and +y up, matching the
262
+ Effect House 720x1280 Screen Transform convention.
263
+ - `anchors`, `offsets`, `pivot`: forwarded to the owned `ScreenTransform`.
264
+ - `opacity`, `color`, `flipX`, `flipY`, `renderOrder`, `transparent`, `alphaTest`, `overlay`, `visible`.
265
+
266
+ Effect House transfer rule: Screen Transform `Size` is the screen-space
267
+ bounding rectangle, not the image's natural pixel size. The image then fits
268
+ inside that rectangle according to `stretchMode`. Fit/fill/stretch changes the
269
+ texture content inside the rectangle; it must not change the object's rigid
270
+ transform. Rotation rotates the whole rectangle around `pivot`.
271
+
272
+ Settings:
273
+
274
+ - `DEFAULT_SCREEN_IMAGE_SETTINGS.transformation`: coordinate space, canvas size,
275
+ position, size, rotation, pivot, render order, overlay/depth behavior, and visibility.
276
+ - `DEFAULT_SCREEN_IMAGE_SETTINGS.image`: source kind, src, fit, opacity, color,
277
+ flips, transparency, and alpha test.
278
+ - The component source union is intentionally small: `default`, static `image`,
279
+ and an already-loaded Three `texture`. Video/GIF/sequence providers resolve
280
+ their media to a live `THREE.Texture` and pass it as `kind: "texture"`.
281
+
282
+ Use `<ScreenImage>` instead of hand-authoring `<ScreenTransform>` + `<mesh>`
283
+ when the object is conceptually a 2D screen image. Use the lower-level mesh path
284
+ only when you need custom geometry or material behavior.
285
+
286
+ ### Shared debug authoring surface for `ScreenImage`
287
+
288
+ The SDK ships a dev-only inspector at `lib/debug-ui/screen-image-settings` for
289
+ authoring `ScreenImage` settings in creator tooling and the v2 template
290
+ preview. It exposes the image renderer, transform, material, and fit controls
291
+ as one shared panel. End-user shipping surfaces should not include the debug
292
+ chrome.
293
+
294
+ The panel's transform copy button copies a JSON payload for the selected object
295
+ including `name`, `kind: "screenImage"`, `transform`, `image`, `material`, and
296
+ `screenSpaceUI`. The `transform` includes `coordinateSpace` and `canvasSize`.
297
+ Use that payload as the handoff from human layout tuning back to code.
298
+
299
+ ### ScreenImage contract for every caller
300
+
301
+ `ScreenImage` is the canonical screen-image object in SDK and client apps. Any
302
+ debug panel, hierarchy row, editor, or runtime controller that targets one must
303
+ operate on the same mounted object state:
304
+
305
+ - Keep one source of truth for `transform`, source/material settings, and
306
+ visibility. Panel edits must immediately update the rendered object.
307
+ - Always wire `onScreenTransformChange` for inspector or editor usage. It is the
308
+ real drag/place path; do not implement a panel-only shadow transform.
309
+ - Use `coordinateSpace="canvas"` plus `canvasSize` for authoring tools, then
310
+ map canvas values to `position`/`size`; never treat design pixels as world
311
+ units. `fit`/`stretchMode` changes content fitting, not the object's bounds.
312
+ - Source replacement must preserve the object identity when possible, stop and
313
+ dispose the previous owned provider, and keep the current transform/material
314
+ settings. Dynamic video/canvas textures must continue marking fitted clones
315
+ for GPU upload.
316
+ - Respect capability-gated controls: static images expose image/material
317
+ controls only; animated providers expose only controls they support. Loading,
318
+ error, retry, pause, ended, and cleanup state must be visible to diagnostics.
319
+ - `visible`, `overlay`, `renderOrder`, `transparent`, `alphaTest`, blend mode,
320
+ opacity, tint, flips, pivot, rotation, size, and position are all part of the
321
+ live object contract and must remain synchronized from SDK or client entry
322
+ points.
323
+
87
324
  ### `<SpriteAnimation sheet={{ url, cols, rows, frames, fps }}>` — Animated transparent overlay
88
325
 
89
326
  The primitive for an animated transparent overlay (a frame border, a gesture-instruction graphic, an animated sticker). One transparent WebP atlas of `cols`×`rows` cells; the first `frames` play left-to-right, top-to-bottom at `fps`. It is a single static GPU texture with a per-frame UV offset — no video decode — so it works in every native and in-app browser and costs ~zero heat.
@@ -115,9 +352,223 @@ import { SpriteAnimation, ScreenSpaceUI } from "@vincentt-xr/sdk";
115
352
 
116
353
  Mount the tracker component; it registers the underlying ML model automatically. No `registerXRPipeline` call needed.
117
354
 
355
+ ### 3D face warp — FBX head binding and face proportion deformation
356
+
357
+ Use `faceWarp3D` on `<CanonicalFaceMeshRenderer>` to deform the live canonical
358
+ face mesh before projection. Defaults are no-op; pass `FACE_WARP_3D_PRESET` for
359
+ a visible starter look, or provide partial settings.
360
+
361
+ ```tsx
362
+ import { CanonicalFaceMeshRenderer, FACE_WARP_3D_PRESET } from "@vincentt-xr/sdk/face-effects";
363
+
364
+ <CanonicalFaceMeshRenderer
365
+ materialType="shader"
366
+ shaderPreset="originalFace"
367
+ textureMap="none"
368
+ alphaMap="canonical"
369
+ alphaOpacity={1}
370
+ baseColor="#ffffff"
371
+ opacity={1}
372
+ side="front"
373
+ faceAlphaMaskUrl="https://cdn.vincentt.studio/assets/sdk/face/v1/Canonical_Face_Color_Mask.png"
374
+ faceWarp3D={FACE_WARP_3D_PRESET}
375
+ />;
376
+ ```
377
+
378
+ Settings (`FaceWarp3DSettings`) include `faceWidth`, `faceHeight`,
379
+ `foreheadHeight`, `cheekWidth`, `eyeSize`, `eyeDistance`, `noseSize`,
380
+ `noseLength`, `mouthWidth`, `mouthHeight`, `jawWidth`, `chinHeight`, plus
381
+ `enabled` and `strength`. Setting values are normalized around zero; typical UI
382
+ sliders map `-100..100` to `-1..1`.
383
+
384
+ For local tuning, import `FaceWarp3DSettingsPanel` from
385
+ `@vincentt-xr/sdk/debug-ui`. It is a draggable debug panel and should not
386
+ be included in the final production scene.
387
+
388
+ ### `FaceMeshRenderer` — scene-object live 3D face mesh
389
+
390
+ Use `FaceMeshRenderer` for the standard SDK face-mesh scene-object component.
391
+ It owns face tracking internally through `FaceTracker`, reads live landmarks,
392
+ deforms the canonical face mesh, and renders through normal face-mesh material
393
+ settings. Prefer this over direct `CanonicalFaceMeshRenderer` in new
394
+ scene-object/editor integrations.
395
+
396
+ Import runtime/material symbols from `@vincentt-xr/sdk/face-effects`:
397
+
398
+ ```tsx
399
+ import {
400
+ DEFAULT_FACE_MESH_RENDERER,
401
+ DEFAULT_FACE_MESH_TRANSFORM,
402
+ FACE_MESH_MATERIAL_TYPES,
403
+ FACE_MESH_SIDES,
404
+ FaceMeshRenderer,
405
+ mapFaceMeshRendererSettingsToProps,
406
+ resolveFaceMeshRendererSettings,
407
+ type FaceMeshRendererSettings,
408
+ } from "@vincentt-xr/sdk/face-effects";
409
+ ```
410
+
411
+ Import scene-object defaults and store helpers from
412
+ `@vincentt-xr/sdk/scene-object`:
413
+
414
+ ```tsx
415
+ import {
416
+ createDefaultComponent,
417
+ createDefaultFaceMeshRendererSettings,
418
+ DEFAULT_RENDER_GROUP_IDS,
419
+ type RenderGroupSceneObjectStore,
420
+ } from "@vincentt-xr/sdk/scene-object";
421
+ ```
422
+
423
+ `FaceMeshRendererSettings` is the serialized component settings contract:
424
+ `enabled`, `faceIndex`, `targetFps`, `showLeftEyeGeometry`,
425
+ `showRightEyeGeometry`, `showMouthGeometry`, `showNoseGeometry`,
426
+ `showOverlayMask`, `showMouthMask`, `showLeftEyeMask`, `showRightEyeMask`,
427
+ `showNoseMask`, `externalMeshUrl`, `externalScale`, `materialType`,
428
+ `shaderPreset`, `textureMap`, `alphaMap`, `alphaOpacity`, `side`, `blendMode`,
429
+ `blendOpacity`, `baseColor`, `opacity`, `compositeShading`,
430
+ `maskTextureSource`, `maskSourceUrl`, `maskColor`, `maskOpacity`,
431
+ `faceTextureUrl`, `faceTextureStyle2Url`, `faceAlphaMaskUrl`, and
432
+ `renderOrder`.
433
+
434
+ `maskTextureSource` is `"solid" | "gradient" | "imageUrl" | "mediaUrl"`.
435
+ Solid and gradient masks use `maskColor`; image/media masks use
436
+ `maskSourceUrl`. `maskOpacity` is always `0..1`.
437
+
438
+ `materialType: "shader"` has three shader presets: `"basicShader"`,
439
+ `"originalFace"`, and `"composite"`. Use
440
+ `resolveFaceMeshRendererSettings(current, patch)` when editing settings so
441
+ shader/material/texture state stays synchronized. Selecting composite applies
442
+ the SDK composite defaults (`textureMap: "canonical"`, hard-light blend,
443
+ `blendOpacity: 0.85`, `compositeShading: true`) instead of silently falling
444
+ back to original-face rendering.
445
+
446
+ `externalMeshUrl` is for canonical-compatible OBJ assets that match the
447
+ renderer's live landmark deformation path. Third-party Face Mesh OBJ files are
448
+ reference assets only until the SDK has an explicit topology mapper from those
449
+ vertices to the 468 tracked face landmarks. Do not bundle OBJ/FBX/PNG reference
450
+ files in `lib/` or examples unless runtime code consumes them correctly.
451
+
452
+ Do not add settings that are not runtime-effective. Skull/ear geometry,
453
+ expression multiplier, and UV1 mapping require parser/runtime support before
454
+ they become public SDK settings.
455
+
456
+ Scene-object example:
457
+
458
+ ```tsx
459
+ const objectId = "face-mesh";
460
+ const transform = createDefaultComponent("Transform", objectId);
461
+ transform.settings.position.y = DEFAULT_FACE_MESH_TRANSFORM.position.y;
462
+
463
+ const faceMesh = createDefaultComponent("FaceMeshRenderer", objectId);
464
+ faceMesh.settings = {
465
+ ...createDefaultFaceMeshRendererSettings(),
466
+ materialType: FACE_MESH_MATERIAL_TYPES.wireframe,
467
+ side: FACE_MESH_SIDES.double,
468
+ baseColor: "#00e5ff",
469
+ opacity: 0.82,
470
+ } satisfies FaceMeshRendererSettings;
471
+
472
+ const store: RenderGroupSceneObjectStore = {
473
+ objects: {
474
+ [objectId]: {
475
+ id: objectId,
476
+ name: "Face Mesh",
477
+ enabled: true,
478
+ renderGroupId: DEFAULT_RENDER_GROUP_IDS.scene3d,
479
+ componentIds: [transform.id, faceMesh.id],
480
+ },
481
+ },
482
+ components: {
483
+ [transform.id]: transform,
484
+ [faceMesh.id]: faceMesh,
485
+ },
486
+ renderGroups: {
487
+ [DEFAULT_RENDER_GROUP_IDS.scene3d]: {
488
+ id: DEFAULT_RENDER_GROUP_IDS.scene3d,
489
+ name: "3D Scene",
490
+ domain: "scene3d",
491
+ enabled: true,
492
+ objectIds: [objectId],
493
+ },
494
+ },
495
+ renderGroupOrder: [DEFAULT_RENDER_GROUP_IDS.scene3d],
496
+ };
497
+ ```
498
+
499
+ When `SceneObjectRenderer` sees both `Transform` and `FaceMeshRenderer` on the
500
+ same object, it passes the authored `Transform` into `FaceMeshRenderer`.
501
+ FaceMesh does not use normal world-space `Transform3D` semantics. It applies
502
+ the supported Transform values in MediaPipe landmark space before camera
503
+ projection; it must never wrap the already-projected mesh in a parent
504
+ `Transform3D`, because that causes depth-dependent screen skew.
505
+
506
+ Supported FaceMesh Transform values are position and uniform scale only.
507
+ Rotation and non-uniform scale are ignored by the runtime until the SDK has a
508
+ real object-space face solve. Authored `position.y` is Y-up, but MediaPipe image
509
+ space is Y-down, so the renderer subtracts Y while adding X/Z.
510
+
511
+ This is the SDK's strict FaceMesh contract: FaceInfo/FaceBinding pose first,
512
+ FaceMesh deformation in face landmark space, supported authored Transform in
513
+ that space, then world/camera projection.
514
+ `DEFAULT_FACE_MESH_TRANSFORM` starts from `DEFAULT_TRANSFORM_3D` and sets
515
+ `position.y` to `0.001` as an intentional FaceMesh authoring bias, not as a
516
+ projection correction. Author a Transform with `position.y = 0` to opt out.
517
+
518
+ `mapFaceMeshRendererSettingsToProps(settings)` maps serialized
519
+ `FaceMeshRendererSettings` to low-level `CanonicalFaceMeshRendererProps`.
520
+ Use it only when bridging custom runtime code; ordinary scene-object rendering
521
+ should use `SceneObjectRenderer`.
522
+
523
+ `CanonicalFaceMeshRenderer` remains exported from `@vincentt-xr/sdk/face-effects`
524
+ for backwards compatibility and low-level experiments, but new editor and
525
+ scene-object work should use `FaceMeshRenderer`.
526
+
527
+ ### `FaceRetouch` — legacy landmark skin retouch
528
+
529
+ Use `FaceRetouch` from `@vincentt-xr/sdk/face-effects` for the legacy
530
+ landmark-masked skin retouch shader. The public controls are intentionally
531
+ minimal:
532
+
533
+ ```tsx
534
+ import {
535
+ FaceRetouch,
536
+ patchFaceRetouchControls,
537
+ resolveFaceRetouchControls,
538
+ type FaceRetouchControls,
539
+ } from "@vincentt-xr/sdk/face-effects";
540
+ import { FaceRetouchSettingsPanel } from "@vincentt-xr/sdk/debug-ui";
541
+
542
+ const [controls, setControls] = useState(() => resolveFaceRetouchControls());
543
+
544
+ const updateControls = (next: Partial<FaceRetouchControls>) => {
545
+ setControls((current) => patchFaceRetouchControls(current, next));
546
+ };
547
+
548
+ <FaceRetouch {...controls} />;
549
+ <FaceRetouchSettingsPanel value={controls} onChange={updateControls} />;
550
+ ```
551
+
552
+ `FaceRetouchControls` has only `skinSmoothing`, `skinRealism`,
553
+ `blemishReduction`, and `previewMask`. Do not invent provider, quality,
554
+ redness, teeth, eye, or blur controls for this component.
555
+
556
+ For multiple people, render one `FaceRetouch` per face:
557
+
558
+ ```tsx
559
+ <FaceRetouch {...controls} faceIndex={0} numFaces={2} />
560
+ <FaceRetouch {...controls} faceIndex={1} numFaces={2} />
561
+ ```
562
+
563
+ `numFaces` defaults to at least `faceIndex + 1`. `clipMinX` and `clipMaxX` are
564
+ optional horizontal UV clipping props in `0..1` space, useful for split-screen
565
+ comparison demos.
566
+
118
567
  ### `<FaceTracker>` — Face tracking (468 landmarks)
119
568
 
120
- Detects faces and provides face transform to children via context. Supports `faceIndex` (default 0) and `scaleWithFace`.
569
+ Detects faces and provides face transform to children via context. Supports
570
+ `faceIndex` (default `0`), `scaleWithFace`, and `followFaceRoll` (default
571
+ `true`; set `false` to keep children upright while the head tilts).
121
572
 
122
573
  ```tsx
123
574
  import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
@@ -132,6 +583,274 @@ import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
132
583
  </FaceTracker>;
133
584
  ```
134
585
 
586
+ ### `FaceInfo` — face-driven effect logic
587
+
588
+ Use `FaceInfo` for head-pose, expression, landmark, bounds, and face-scale logic.
589
+ Do not read raw MediaPipe results or repeat landmark indices unless the facade
590
+ does not expose data required by a research/debugging tool.
591
+
592
+ ```tsx
593
+ import { hasFaceExpression, useFaceInfo } from "@vincentt-xr/sdk/tracking";
594
+
595
+ function FaceLogic() {
596
+ const face = useFaceInfo({ active: true });
597
+ if (!face) return null;
598
+
599
+ const mouthOpen = hasFaceExpression(face, "jawOpen");
600
+ const lookingLeft = face.pose.rotation.y > 0.2;
601
+ return <group visible={mouthOpen} rotation-y={lookingLeft ? 0.2 : 0} />;
602
+ }
603
+ ```
604
+
605
+ | Field | Coordinate contract |
606
+ | --------------- | ---------------------------------------------------------------- |
607
+ | `index` | Current result-array index; not a persistent identity |
608
+ | `landmarks` | MediaPipe normalized tracker-input coordinates, top-left origin |
609
+ | `bounds` | Normalized tracker-input bounds, top-left origin |
610
+ | `pose.position` | Existing face-tracker Three.js world position |
611
+ | `pose.rotation` | Radians: X pitch (+up), Y yaw (+left), Z roll |
612
+ | `pose.scale` | Smoothed uniform sizing heuristic used by face anchors/occluders |
613
+ | `eyeDistance` | Outer-eye distance in normalized tracker-input coordinates |
614
+ | `blendshapes` | MediaPipe category names and scores in `[0, 1]` |
615
+
616
+ `useFaceInfo({ faceIndex })` is the normal single-face React reader.
617
+ `useFaceInfos({ numFaces })` publishes all current faces. Both acquire the shared
618
+ face model through the SDK registry and use the existing throttle/empty-frame
619
+ hold lifecycle; do not mount a separate result polling loop. Call them inside
620
+ `XRProvider` and the R3F Canvas. Options are `active`, `numFaces`, `targetFps`,
621
+ `throttleMs`, and `holdMs`; `useFaceInfo` additionally accepts `faceIndex`.
622
+
623
+ ```tsx
624
+ function MultiFaceLogic() {
625
+ const faces = useFaceInfos({ numFaces: 3 });
626
+ return <group visible={faces.some((face) => hasFaceExpression(face, "jawOpen"))} />;
627
+ }
628
+ ```
629
+
630
+ For `useFrame` code that already has a low-level face node, call
631
+ `getFaceInfo(node, faceIndex, target)`. Create `target` once with
632
+ `createFaceInfoTarget()` and reuse it to avoid per-frame allocations. Use
633
+ `getFaceInfo` and `createFaceInfoTarget` from `@vincentt-xr/sdk/low-level`; they
634
+ are node-in escape hatches, not the normal React API. Use `getFaceBlendshape`
635
+ and `hasFaceExpression`; absent scores resolve to `0`, and the default expression
636
+ threshold is `DEFAULT_FACE_EXPRESSION_THRESHOLD`. Bind the R3F camera once with
637
+ `node.setCamera(camera, faceIndex)` before reading pose; the React hooks do this
638
+ automatically.
639
+
640
+ All supported expression names are typed by `FaceBlendshapeName` and listed in
641
+ the greppable `FACE_BLENDSHAPE_NAMES` constant exported from
642
+ `@vincentt-xr/sdk/tracking`. This includes `eyeBlinkLeft`, `eyeBlinkRight`,
643
+ `eyeWideLeft`, `eyeWideRight`, and `jawOpen`; do not guess string names.
644
+
645
+ Use `normalizedToScreenPixels` with the tracker input/display viewport size to
646
+ display a face landmark in Screen Image/Text/Shape coordinates. The tracker
647
+ input is already the SDK feed; do not apply media cover/mirroring a second time.
648
+ Never mutate or mirror `FaceInfo.landmarks` in place, manually project them,
649
+ infer a persistent face ID, or copy numeric landmark indices/thresholds into
650
+ application code.
651
+
652
+ ```ts
653
+ const nose = normalizedToScreenPixels({
654
+ point: face.landmarks[FACE_LANDMARK_INDICES["face.noseTip"]],
655
+ viewportSize,
656
+ });
657
+ ```
658
+
659
+ Effect House mapping: `FaceInfo.landmarks` replaces `pointsArray`, `bounds`
660
+ replaces `rect`, `pose.rotation` provides pitch/yaw/roll, and `eyeDistance` is
661
+ named the same. Use Face Actions below for the semantic `FaceAction` role.
662
+ Vincentt uses MediaPipe's face topology and top-left origin; it is not a
663
+ 106-point or bottom-left-compatible alias.
664
+
665
+ Run the example route `/face-info` to verify all landmarks, live bounds, nose
666
+ position, pose, eye distance, and `jawOpen` using `ScreenShape`, `ScreenText`,
667
+ and `Axes`.
668
+
669
+ ### `useFaceDetection()` — face presence lifecycle and count
670
+
671
+ Import `useFaceDetection` from `@vincentt-xr/sdk/tracking` when logic needs face
672
+ detected/lost edges, selected-face presence, or the current face count. It wraps
673
+ the shared `useFaceInfos` reader and does not start another model.
674
+
675
+ ```tsx
676
+ const detection = useFaceDetection({ faceIndex: 0, numFaces: 5 });
677
+
678
+ if (detection.justDetected) showEffect();
679
+ if (detection.detected) updateEffect(detection.face);
680
+ if (detection.justLost) hideEffect();
681
+ ```
682
+
683
+ | Visual-scripting term | SDK field |
684
+ | ------------------------------------- | ------------------------------------ |
685
+ | Face Detection Begin / face appeared | `justDetected`, `phase === "begin"` |
686
+ | Face Detection Stay / face present | `detected`, `phase === "stay"` |
687
+ | Face Detection End / face disappeared | `justLost`, `phase === "end"` |
688
+ | Face Detection None / no face | `!detected`, `phase === "none"` |
689
+ | Face Count / number of faces | `faceCount` and `faces.length` |
690
+ | Which Face / selected face | `faceIndex` option and `face` result |
691
+
692
+ The full result is `face`, `faces`, `faceCount`, `detected`, `justDetected`,
693
+ `justLost`, and `phase`. Options are the normal `useFaceInfos` options plus
694
+ `faceIndex`: `active`, `numFaces`, `targetFps`, `throttleMs`, and `holdMs`.
695
+ Default `holdMs: 250` prevents a transient missed frame from firing `end`; use
696
+ `holdMs: 0` only when immediate disappearance is required. Do not compare raw
697
+ result arrays or build a local previous-frame ref. The verified
698
+ `/face-detection` page tests five selectable face slots and all four phases; its
699
+ real-camera lifecycle flow passed on 26 July 2026.
700
+
701
+ ### Reusable normalized and face image crops
702
+
703
+ Import `drawNormalizedCrop` from `@vincentt-xr/sdk` for any normalized image
704
+ region. Import `drawFaceCrop` from `@vincentt-xr/sdk/tracking` when the bounds
705
+ come from `FaceInfo`. Both preserve the requested output pixel aspect ratio,
706
+ clamp at source edges, and draw into a caller-owned canvas without starting a
707
+ tracker or allocating per call.
708
+
709
+ ```ts
710
+ const cropCanvas = document.createElement("canvas");
711
+
712
+ const drawn = drawFaceCrop({
713
+ source: video,
714
+ face,
715
+ canvas: cropCanvas,
716
+ outputSize: { width: 224, height: 224 },
717
+ padding: 0.2,
718
+ });
719
+ ```
720
+
721
+ Bounds use normalized tracker-input/source coordinates with a top-left origin.
722
+ `padding` is the fraction of detected width/height added on each side. The face
723
+ helper defaults to `0.2`; the generic helper defaults to `0`. The functions do
724
+ not mirror, classify, upload, or retain pixels. A `false` result means the
725
+ source dimensions, bounds, output dimensions, or canvas context were unusable.
726
+ Reuse one canvas on frame-driven paths. Face attribute expression should feed
727
+ this crop into its independent image classifier; never derive emotion labels
728
+ from `FaceInfo.blendshapes`.
729
+
730
+ ### Face Actions — semantic face effect logic
731
+
732
+ Import `FACE_ACTION_NAMES`, `DEFAULT_FACE_ACTION_OPTIONS`,
733
+ `getFaceActionValue`, `hasFaceAction`, and `useFaceAction` from
734
+ `@vincentt-xr/sdk/tracking`. They derive from caller-owned `FaceInfo`; they do
735
+ not start a second tracker.
736
+
737
+ ```tsx
738
+ const face = useFaceInfo();
739
+ const blink = useFaceAction(face, "eyeBlink");
740
+
741
+ useEffect(() => {
742
+ if (blink.justStarted) capturePhoto();
743
+ }, [blink.justStarted]);
744
+
745
+ const mouthAmount = face ? getFaceActionValue(face, "mouthOpen") : 0;
746
+ ```
747
+
748
+ | Action | Source and direction | Default start / release / hold |
749
+ | ------------------------------------------------------------- | --------------------------------- | ------------------------------ |
750
+ | `browRaise` | max brow-up blendshape | `0.45 / 0.30 / 50 ms` |
751
+ | `eyeBlink`, `eyeBlinkLeft`, `eyeBlinkRight`, `eyeBlinkEither` | typed eye-blink blendshapes | `0.55 / 0.40 / 50 ms` |
752
+ | `mouthOpen` | `jawOpen` | `0.45 / 0.30 / 50 ms` |
753
+ | `mouthPout` | `mouthPucker` | `0.50 / 0.35 / 50 ms` |
754
+ | `headUp`, `headDown` | positive/negative pitch magnitude | `10° / 6° / 80 ms` |
755
+ | `headTurnLeft`, `headTurnRight` | positive/negative yaw magnitude | `10° / 6° / 80 ms` |
756
+ | `headTiltLeft`, `headTiltRight` | positive/negative roll magnitude | `10° / 6° / 80 ms` |
757
+
758
+ Blendshape values are `[0, 1]`; head values and thresholds are radians. Left
759
+ and up use the SDK-positive pose convention. Directional actions always return
760
+ a non-negative magnitude. Pass `neutralRotation: { ...face.pose.rotation }`
761
+ captured from an explicit user calibration gesture when the resting pose should
762
+ be treated as zero. Without it, head actions use zero rotation.
763
+
764
+ Use `getFaceActionValue` for continuous animation, `hasFaceAction` for a simple
765
+ level check, and `useFaceAction` for `active`, `justStarted`, `justEnded`, and
766
+ `phase: "begin" | "stay" | "end" | "none"`
767
+ with SDK hysteresis and hold timing. Do not guess blendshape strings, flip pose
768
+ signs, or rebuild local debounce/hysteresis logic. For multiple faces, select
769
+ one with `useFaceInfo({ faceIndex })`, or call
770
+ `getStrongestFaceAction(faces, action)` for an "any face" effect, then pass its
771
+ `face` into `useFaceAction`. The strongest selector is one pass and starts no
772
+ additional reader.
773
+
774
+ Effect House-style physical movement maps as follows: left/right/both/either
775
+ blink → `eyeBlinkLeft` / `eyeBlinkRight` / `eyeBlink` / `eyeBlinkEither`;
776
+ mouth open → `mouthOpen`; pout → `mouthPout`; eyebrow movement → `browRaise`.
777
+ Semantic blink left/right is viewer-facing and therefore swaps MediaPipe's raw
778
+ anatomical-side blendshape names. Never swap `FaceInfo.blendshapes` itself.
779
+ Every detected movement emits at least one `stay` frame after `begin` before
780
+ `end`.
781
+ These are observable movements, not image classification. Do not infer
782
+ `happy`, `sad`, `angry`, `fear`, `disgust`, `surprise`, or `neutral` from local
783
+ threshold combinations. Use the separate Face Attribute Expression API below.
784
+
785
+ Run `/face-actions` to inspect every live value, set neutral pose, tune the
786
+ selected action, and repeat the all-actions checklist. All actions and the
787
+ default thresholds were verified with the real-camera page on 26 July 2026.
788
+ Run the verified `/facial-movement-expression` page to test the Effect
789
+ House-compatible physical movement subset, phase edges, fixed-face or
790
+ strongest-face selection, and the image classifier. Its complete real-camera
791
+ flow passed on 26 July 2026.
792
+
793
+ ### Face Attribute Expression — image-based classification
794
+
795
+ Import `useFaceAttributeExpression`, `FACE_ATTRIBUTE_EXPRESSION_NAMES`, and
796
+ their types from `@vincentt-xr/sdk/tracking`. Pass caller-owned `FaceInfo`; the
797
+ hook reuses the shared session input frame, crops that face, and lazy-loads the
798
+ OpenCV MobileFaceNet INT8 ONNX classifier. It does not create another face
799
+ reader or detector.
800
+
801
+ ```tsx
802
+ const face = useFaceInfo();
803
+ const attribute = useFaceAttributeExpression(face, {
804
+ targetFps: 5,
805
+ minConfidence: 0.55,
806
+ });
807
+
808
+ if (attribute.status === "ready") {
809
+ const { expression, confidence, scores } = attribute.result;
810
+ }
811
+ ```
812
+
813
+ The lifecycle is `idle | loading | ready | error`. Ready results contain
814
+ `expression`, `confidence`, and `scores`. `expression` can be `angry`,
815
+ `disgust`, `fear`, `happy`, `neutral`, `sad`,
816
+ `surprise`, or `unknown`. `scores` contains only the seven real model classes;
817
+ there is no fabricated `unknown` score. `unknown` is the low-confidence fallback
818
+ below `minConfidence`. Default `targetFps` is `5`; the model is about 1.36 MB,
819
+ loads outside the main bundle, and is SHA-384 verified. ONNX Runtime's WASM
820
+ worker files load from the jsDelivr package path that matches the active
821
+ `onnxruntime-web` runtime version, not through the consuming app's bundler. The
822
+ runtime files are fetched from jsDelivr without SDK-managed SRI because ONNX
823
+ Runtime resolves its own WASM assets.
824
+
825
+ For best threaded SIMD performance, production pages that run expression
826
+ classification should be cross-origin isolated
827
+ (`Cross-Origin-Opener-Policy: same-origin` and
828
+ `Cross-Origin-Embedder-Policy: require-corp`) so `SharedArrayBuffer` is
829
+ available. Without those headers, ONNX Runtime falls back to single-threaded
830
+ WASM.
831
+
832
+ `modelUrl` overrides must accept float32 NCHW `[1, 3, 112, 112]`, with planar
833
+ RGB normalized to `[-1, 1]`, and return seven logits ordered `angry`, `disgust`,
834
+ `fear`, `happy`, `neutral`, `sad`, `surprise`. Do not assume an arbitrary
835
+ MobileFaceNet ONNX model is compatible. Failed loads are removed from the
836
+ session cache so a remounted consumer can retry.
837
+
838
+ This is a visual-expression estimate, not ground-truth emotion, intent,
839
+ identity, or mental state. Never replace it with blendshape heuristics. Run the
840
+ verified `/facial-movement-expression` page to inspect every live model score.
841
+
842
+ ### `<Axes>` — generic 3D orientation gizmo
843
+
844
+ Use `Axes` to visualize any 3D pose. X is red, Y is green, and Z is blue. Keep
845
+ tracking separate: bind a face, hand, or object pose through standard Three.js
846
+ `position` and `rotation` props.
847
+
848
+ ```tsx
849
+ import { Axes } from "@vincentt-xr/sdk";
850
+
851
+ <Axes size={0.38} position={pose.position} rotation={pose.rotation} />;
852
+ ```
853
+
135
854
  ### `<FaceTracker.Mesh />` / `<FaceMesh>` — Deforming 468-point face mesh
136
855
 
137
856
  For masks, makeup, and full-face decoration that deforms with skin.
@@ -154,6 +873,10 @@ import { HandTracker } from "@vincentt-xr/sdk/tracking";
154
873
  <HandTracker />;
155
874
  ```
156
875
 
876
+ Supports multiple detected hands: `hand` selects `"left"` or `"right"` when that
877
+ hand is available; `handIndex` falls back to positional selection. Existing
878
+ one-hand scenes keep working via the default first-hand selection.
879
+
157
880
  ### `<GestureTracker>` — Named gestures
158
881
 
159
882
  The gesture door — the component that recognizes named gestures. Its `node.gesture` is one of these exact strings (copy verbatim — these are the only valid values):
@@ -232,6 +955,12 @@ Every tracker takes an `active` prop (default `true`). Set `active={false}` and
232
955
 
233
956
  When several consumers share one model, it runs while **any** of them is active. `active` gates whether the model runs; `perf.targetFps` (below) caps how often it runs while active — use both. `<Segmentation>` also takes `active` (distinct from its `visible` prop, which only gates mask exposure while inference keeps running).
234
957
 
958
+ Session lifecycle preserves that ownership state: `session.stop()` pauses every
959
+ tracker without releasing its MediaPipe resources, and the next `session.start()`
960
+ resumes only models that still have at least one active owner. Static-image
961
+ detection pulses use the same suspension path, so source switches do not
962
+ accidentally reactivate an `active={false}` tracker.
963
+
235
964
  ### `perf` — cap inference rate + camera resolution
236
965
 
237
966
  Set `perf` on `<XRProvider>` to trade tracking fidelity for lower load and heat. This is the single biggest lever if the device runs hot.
@@ -257,6 +986,11 @@ Per-tracker override beats the session default:
257
986
 
258
987
  The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — good for both dev laptops and end-user phones. Don't lower it further without a reason; don't remove it to "get full quality" unless you know the target device can take the heat.
259
988
 
989
+ SDK-owned tracker, occluder, and canonical-face assets are SHA-384 verified
990
+ against hashes pinned in the package. A missing hash for
991
+ `https://cdn.vincentt.studio` fails closed; custom consumer-owned asset URLs
992
+ remain best-effort and warn when no pinned hash exists.
993
+
260
994
  ---
261
995
 
262
996
  ## Reading tracker state
@@ -265,9 +999,11 @@ The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — g
265
999
 
266
1000
  Wraps children; the child's world position is updated to the tracked landmark each frame. Use for all decoration that simply follows a face/hand/body point.
267
1001
 
268
- Valid targets:
1002
+ Valid targets are centralized in `FACE_TARGETS`; use that exported list for
1003
+ editor dropdowns instead of copying it. Left/right mean the tracked person's
1004
+ perspective.
269
1005
 
270
- - **Face:** `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`
1006
+ - **Face binding presets:** `face.center`, `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEye`, `face.rightEye`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`, `face.leftEarlobe`, `face.rightEarlobe`, `face.custom`
271
1007
  - **Hand:** `hand.wrist`, `hand.thumbTip`, `hand.indexTip`, `hand.middleTip`, `hand.ringTip`, `hand.pinkyTip`, `hand.indexBase`, `hand.middleBase`, `hand.ringBase`, `hand.pinkyBase`
272
1008
  - **Body:** `body.head`, `body.neck`, `body.leftShoulder`, `body.rightShoulder`, `body.leftElbow`, `body.rightElbow`, `body.leftWrist`, `body.rightWrist`, `body.leftHip`, `body.rightHip`, `body.leftKnee`, `body.rightKnee`, `body.leftAnkle`, `body.rightAnkle`
273
1009
 
@@ -282,6 +1018,32 @@ Valid targets:
282
1018
  </FaceTracker>
283
1019
  ```
284
1020
 
1021
+ `face.center` is the general default. Eye targets are composite eye centers;
1022
+ `*EyeOuter` targets are exact outer-corner landmarks. Earlobe targets are the
1023
+ nearest stable MediaPipe face-oval points, not anatomical earlobe detections.
1024
+ All Effect House `FaceBindingAnchorType` names therefore have a Vincentt
1025
+ equivalent, but their numeric canonical-mesh coordinates are not compatible.
1026
+
1027
+ For `face.custom`, pass normalized face-local coordinates. The origin is face
1028
+ center; X runs from person's left `-1` to right `+1`, Y from chin `-1` to
1029
+ forehead `+1`, and positive Z comes toward the camera.
1030
+
1031
+ ```tsx
1032
+ <FaceTracker>
1033
+ <TrackingAnchor target="face.custom" customAnchorPoint={{ x: 0.7, y: 0.2, z: 0 }} smoothing={15}>
1034
+ <mesh />
1035
+ </TrackingAnchor>
1036
+ </FaceTracker>
1037
+ ```
1038
+
1039
+ Scene objects use the same `TrackingAnchor` component settings. A 2D null with
1040
+ `ScreenTransform` + `TrackingAnchor` projects the preset into SDK design pixels,
1041
+ so all child Screen Shapes move as one group. Use `/face-binding-anchors` to
1042
+ test every preset on the editable `rose null`; the inspector target options
1043
+ come directly from `FACE_TARGETS`. Toggle `Follow Face Roll` to rotate the
1044
+ attachment with head tilt. The example hierarchy intentionally shows only the
1045
+ combined root object.
1046
+
285
1047
  **`smoothing` — smooth motion from throttled inference.** Inference is throttled (the `perf` API caps it at 24fps by default) while rendering runs at 60-120fps, so a raw anchor holds then jumps on each new sample — visible stepping under head/hand movement. A plain lerp only hides it at rest; under motion it still steps. Pass `smoothing` (`~15` is a good default) and the anchor runs a **One Euro filter** — the standard adaptive low-pass for interactive tracking: it smooths hard when the landmark moves slowly (kills jitter) and loosens as it speeds up (kills lag), and never extrapolates so it can't overshoot or wobble. Higher `smoothing` is snappier (less lag, slightly more jitter); lower is smoother. It snaps (resets the filter) on first placement and on re-acquire, so it never slides in from a stale position. `0` (default) keeps the raw snap. Rotation is not smoothed. Prefer this over a hand-rolled `useFrame` lerp — a hand-rolled lerp reintroduces the stepping.
286
1048
 
287
1049
  ```tsx
@@ -350,10 +1112,101 @@ import { GestureTracker, GestureTrigger } from "@vincentt-xr/sdk/tracking";
350
1112
 
351
1113
  In a v2-template project this is already mounted in `src/App.tsx` — do NOT add a second one there. Mentioned here because `customBackground` is how you do background replacement: pair it with `<Segmentation type="portrait" />` and pass the segmentation mask texture as `segmentationMask`.
352
1114
 
1115
+ `VideoBackground` is a flat camera-background component: its material keeps
1116
+ `depthTest={false}` and `depthWrite={false}`. Editor-style 3D viewports that
1117
+ treat the camera feed as a real scene plane should render a separate editor
1118
+ video plane instead of changing `VideoBackground` semantics.
1119
+
353
1120
  ---
354
1121
 
355
1122
  ## Mesh primitives and textures
356
1123
 
1124
+ `ScreenImage` keeps fitted clones synchronized for dynamic video and canvas textures.
1125
+
1126
+ ### Texture Types API usage
1127
+
1128
+ `ScreenImage` is the common render target for static and animated texture sources. Use
1129
+ the normal runtime import for rendering:
1130
+
1131
+ ```tsx
1132
+ import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
1133
+
1134
+ <ScreenSpaceUI>
1135
+ <ScreenImage
1136
+ source={{ kind: "image", src: imageUrl }}
1137
+ materialType="unlit"
1138
+ stretchMode="fit"
1139
+ transparent
1140
+ />
1141
+ </ScreenSpaceUI>;
1142
+ ```
1143
+
1144
+ Supported source behavior:
1145
+
1146
+ - Image: static URL texture. No playback controls.
1147
+ - Runtime texture: pass `{ kind: "texture", texture }` when supplying an existing `THREE.Texture`.
1148
+ - Video: use a managed `THREE.VideoTexture`; support play/pause, restart, frame stepping, playback rate, forward, and reverse stepping. Negative `playbackRate` is not used.
1149
+ - GIF: decode frames into a canvas texture. Preserve GIF dimensions and per-frame delays before applying ScreenImage fitting.
1150
+ - Image sequence: draw loaded frames into a canvas texture using FPS and frame state.
1151
+
1152
+ Animated sources support forward, reverse, ping-pong, random, looping, finite loop
1153
+ counts, pause/resume at current frame, restart, frame status, and ended state where
1154
+ the source supports it. Static image sources must not expose animation controls.
1155
+
1156
+ ScreenImage inspector rules:
1157
+
1158
+ - `ScreenImageSettingsPanel` must read/write the same scene-object state used by the rendered `ScreenImage`.
1159
+ - Transform drag/place must be wired through `onScreenTransformChange`.
1160
+ - Use `mapScreenImageSettingsToProps` to convert inspector canvas coordinates into render props.
1161
+ - Do not pass inspector pixel dimensions directly as world-space `ScreenImage` dimensions.
1162
+ - Dynamic canvas/video texture updates must mark fitted texture clones for GPU upload.
1163
+
1164
+ Low-level texture integration imports from `@vincentt-xr/sdk/low-level`:
1165
+
1166
+ ```ts
1167
+ import {
1168
+ BasicAnimatedTextureController,
1169
+ createTextureSourceResolver,
1170
+ getPlaybackFrame,
1171
+ getSequenceFrame,
1172
+ type AnimatedTextureController,
1173
+ type AnimatedTexturePlayMode,
1174
+ type ResolvedTextureSource,
1175
+ type TextureSourceResolver,
1176
+ type TextureSourceProvider,
1177
+ } from "@vincentt-xr/sdk/low-level";
1178
+ ```
1179
+
1180
+ Controller lifecycle is: `play()`, `pause()`, `restart()`, optional `seek()`,
1181
+ `setPlayMode("forward" | "reverse" | "ping-pong" | "random")`,
1182
+ `setLoop(enabled, count)`, `subscribe(listener)`, then `dispose()`. Read
1183
+ `getSnapshot()` for state, frame/time, duration, frame count, loop, and errors;
1184
+ check `capabilities` before rendering a control.
1185
+
1186
+ Frame helpers are also available from the same `/low-level` door:
1187
+ `getPlaybackFrame(tick, frameCount, mode, loop, loopCount)` handles forward,
1188
+ reverse, ping-pong, random, loop, and ended math; `getSequenceFrame(elapsedSeconds,
1189
+ fps, frameCount)` maps elapsed time to a bounded sequence index.
1190
+
1191
+ Event compatibility: subscribe with `controller.on(type, listener)` and remove
1192
+ with the returned disposer or `off()`. Supported event types are
1193
+ `KeyFrameInSecond`, `Pause`, `PlayBegin`, `PlayEnd`, `Resume`, and `Stop`.
1194
+ Event snapshots include the current frame/time, playback state, loop settings,
1195
+ and optional duration/frame count. Use capability checks before exposing an
1196
+ event-driven control.
1197
+
1198
+ Consumer example: `docs/texture-api-consumer.md`.
1199
+
1200
+ Browser verification checklist: Chrome/Android (autoplay requires muted media),
1201
+ Safari/iOS (inline video requires `playsInline` and user-gesture fallback), and
1202
+ background tabs (timers may throttle; frame state must remain bounded). Validate
1203
+ source switching, pause-at-current-frame, loop-off ending, reverse/ping-pong,
1204
+ stretch modes, drag/place transforms, and unmount cleanup in each target.
1205
+
1206
+ `ResolvedTextureSource` owns a texture, optional controller, metadata, and
1207
+ ownership-safe `dispose()`. Abort stale loads, stop timers/RAF, remove media
1208
+ listeners, and never dispose externally owned textures.
1209
+
357
1210
  Use R3F intrinsic `<mesh>`, `<group>`, `<planeGeometry>`, `<meshBasicMaterial>`, etc. Load image textures with drei's `useTexture`.
358
1211
 
359
1212
  **Always name your meshes** (`name="myLabel"`). Names enable runtime lookup AND visual-feedback editing by the editor.
@@ -373,29 +1226,790 @@ const tex = useTexture("https://cdn.../badge.webp");
373
1226
 
374
1227
  ---
375
1228
 
1229
+ ## Scene-object runtime and authoring state
1230
+
1231
+ Runtime components import from `@vincentt-xr/sdk`. Scene-object data and store
1232
+ operations import from `@vincentt-xr/sdk/scene-object`. DOM editors import from
1233
+ `@vincentt-xr/sdk/debug-ui`.
1234
+
1235
+ ### `<Transform3D value={settings}>` and `<MeshRenderer value={settings}>`
1236
+
1237
+ `Transform3D` applies position, rotation (degrees), scale, visibility, and optional
1238
+ Drei transform controls to its children. `MeshRenderer` renders a cube, sphere,
1239
+ plane, or first-pass GLTF/GLB asset with basic/standard material settings.
1240
+
1241
+ ```tsx
1242
+ import {
1243
+ DEFAULT_MESH_RENDERER,
1244
+ DEFAULT_TRANSFORM_3D,
1245
+ MeshRenderer,
1246
+ Transform3D,
1247
+ } from "@vincentt-xr/sdk";
1248
+
1249
+ <Transform3D
1250
+ value={{
1251
+ ...DEFAULT_TRANSFORM_3D,
1252
+ position: { x: 0, y: 0, z: -1 },
1253
+ showTransformControls: true,
1254
+ }}
1255
+ onChange={setTransform}
1256
+ >
1257
+ <MeshRenderer
1258
+ value={{
1259
+ ...DEFAULT_MESH_RENDERER,
1260
+ material: { ...DEFAULT_MESH_RENDERER.material, color: "#38bdf8" },
1261
+ }}
1262
+ />
1263
+ </Transform3D>;
1264
+ ```
1265
+
1266
+ `Transform3DProps` is `{ value, onChange?, children }`.
1267
+ `MeshRendererProps` is `{ value, renderOrder? }`. `renderOrder` is applied to the
1268
+ renderable mesh itself. Disabled or invisible settings render nothing.
1269
+
1270
+ `MeshRendererSettings` fields:
1271
+
1272
+ - `enabled`, `visible`: gate rendering.
1273
+ - `mesh`: `"cube" | "sphere" | "plane"` for primitive fallback.
1274
+ - `assetUrl?`: optional GLTF/GLB URL. When present, the asset renders instead of
1275
+ the primitive mesh.
1276
+ - `assetNodeName?`: optional object name inside the loaded GLTF scene. When
1277
+ missing or unresolved, the whole cloned scene renders.
1278
+ - `material.type`: `"standard" | "basic"`.
1279
+ - `material.color`, `opacity`, `wireframe`, `roughness`, `metalness`: material
1280
+ overrides. Wireframe belongs to material state, not geometry state.
1281
+ - `material.renderState?`: optional advanced draw behavior:
1282
+ - `depthTest`, `depthWrite`, `colorWrite`: booleans. `colorWrite` is the
1283
+ all-channel write gate.
1284
+ - `colorMask`: `{ r, g, b, a }` per-channel color write gates. This is
1285
+ applied as scoped WebGL color-mask state for each mesh draw and restored
1286
+ after the object renders.
1287
+ - `depthFunc`: `"never" | "less" | "equal" | "lessEqual" | "greater" |
1288
+ "notEqual" | "greaterEqual" | "always"`.
1289
+ - `cullMode`: `"back" | "front" | "off"`. The default `"back"` maps to
1290
+ Three.js `FrontSide`; `"off"` maps to `DoubleSide`.
1291
+ - `blendMode`: `"normal" | "additive" | "subtractive" | "multiply" |
1292
+ "none"`.
1293
+ - `polygonOffset`: `{ enabled, factor, units }`.
1294
+ - `shadowMode`: `"off" | "cast" | "receive" | "castReceive"`.
1295
+
1296
+ Current GLTF support is intentionally first-pass: the renderer clones the loaded
1297
+ scene and applies color, opacity, wireframe, render-state fields, and shadow
1298
+ flags to cloned mesh materials.
1299
+ Do not rely on it yet for advanced material graph preservation, animation,
1300
+ skinning behavior, or production asset lifecycle guarantees.
1301
+
1302
+ Render-state defaults preserve current behavior: depth test/write on,
1303
+ `lessEqual` depth function, color write on, back-face culling, normal blending,
1304
+ RGBA color mask fully enabled, and polygon offset disabled. Instance count is not
1305
+ part of `MeshRenderer.material.renderState`; it belongs to instanced mesh
1306
+ rendering rather than material shading state.
1307
+
1308
+ `assetUrl` uses Drei `useGLTF`, so it suspends while loading. `MeshRenderer`
1309
+ wraps asset-backed rendering in `Suspense fallback={null}` so data-driven
1310
+ `SceneObjectRenderer` usage is safe by default. Add your own outer `Suspense`
1311
+ or call `useGLTF.preload(url)` only when you want a visible loading state or
1312
+ early fetch. Primitive meshes do not require Suspense.
1313
+
1314
+ ### Screen Transform data and resolver
1315
+
1316
+ Import `ScreenTransform2DSettings`, `DEFAULT_SCREEN_TRANSFORM_2D`,
1317
+ `DEFAULT_SCREEN_TRANSFORM_ROOT`, `DEFAULT_REFERENCE_PIXELS_PER_UNIT`,
1318
+ `mergeScreenTransform2D`, and `resolveScreenTransform2D` from the core door.
1319
+
1320
+ `ScreenTransform2DSettings` owns enabled/visible state, canvas-pixel position and
1321
+ size, normalized pivot, degree rotation, 2D scale, reference pixels per unit,
1322
+ render order, overlay mode, and guide visibility.
1323
+
1324
+ ```ts
1325
+ import {
1326
+ DEFAULT_SCREEN_TRANSFORM_2D,
1327
+ mergeScreenTransform2D,
1328
+ resolveScreenTransform2D,
1329
+ type ScreenTransform2DSettings,
1330
+ } from "@vincentt-xr/sdk";
1331
+
1332
+ const transform: ScreenTransform2DSettings = mergeScreenTransform2D({
1333
+ position: { x: 0, y: 240 },
1334
+ size: { width: 420, height: 420 },
1335
+ showTransformGuides: true,
1336
+ });
1337
+
1338
+ const resolved = resolveScreenTransform2D({
1339
+ transform,
1340
+ parentSize: { width: 9, height: 16 },
1341
+ });
1342
+ ```
1343
+
1344
+ Use `mergeScreenTransform2D` to fill nested defaults. Use
1345
+ `resolveScreenTransform2D` whenever runtime or editor tooling must convert the
1346
+ authoring model into center, size, rotation, pivot offset, and aspect. Do not copy
1347
+ the default `720x1280` root or `32` reference-pixel constant into another renderer
1348
+ or guide.
1349
+
1350
+ ### Screen/world coordinate utilities
1351
+
1352
+ Import the `*ScreenPixels` coordinate helpers, `screenPointToRay`, and
1353
+ `smoothNumber` / `smoothVector2` / `smoothVector3` from `@vincentt-xr/sdk`. Use
1354
+ these shared helpers instead of rewriting mapping or projection math or assuming
1355
+ design pixels are Three.js world units.
1356
+ SDK screen pixels use the Effect House-style `720x1280` design canvas, center
1357
+ origin, and +Y up.
1358
+
1359
+ ```ts
1360
+ const units = screenPixelsToUnits(120, { parentSize: screenDimensions });
1361
+ const pixels = screenUnitsToPixels(units, { parentSize: screenDimensions });
1362
+
1363
+ const labelPosition = worldToScreenPixels({
1364
+ point: trackedObject.getWorldPosition(new THREE.Vector3()),
1365
+ camera,
1366
+ viewportSize,
1367
+ });
1368
+
1369
+ const placementRay = screenPointToRay({
1370
+ point: { x: tapX, y: tapY },
1371
+ camera,
1372
+ viewportSize,
1373
+ });
1374
+ ```
1375
+
1376
+ - Use pixel/unit conversion for Screen Image, Screen Text, Screen Shape, editor
1377
+ handles, and custom `ScreenSpaceUI` content.
1378
+ - Use `worldToScreenPixels` for labels, prompts, and indicators that follow a
1379
+ face, hand, body joint, or 3D object. Its `depth` is projected NDC depth; hide
1380
+ the result when `depth` is outside `[-1, 1]` because it is outside the camera
1381
+ clip range.
1382
+ - Use `screenPointToRay` for tap selection, hit testing, and 3D placement. Pass
1383
+ design-canvas coordinates, not DOM client pixels.
1384
+ - Use `clientToScreenPixels` / `screenPixelsToClient` for pointer events and DOM
1385
+ overlays; pass the viewport element's `getBoundingClientRect()` result.
1386
+ - Use `normalizedToScreenPixels` / `screenPixelsToNormalized` for normalized
1387
+ viewport coordinates with top-left origin. These values are not automatically
1388
+ camera-crop adjusted.
1389
+ - Use `mediaToScreenPixels` / `screenPixelsToMedia` with the shared
1390
+ `XRMediaLayout` for camera/image pixels. They apply cover/contain offsets,
1391
+ scaling, and optional display mirroring; do not manually repeat that math.
1392
+ - Use `smoothNumber`, `smoothVector2`, or `smoothVector3` for tracked animation.
1393
+ Pass the R3F frame `delta` in seconds and a smoothing rate (larger is faster).
1394
+ Reuse the optional `out` vector in per-frame paths to avoid allocations.
1395
+ - Zero-size viewport/parent inputs return neutral zero coordinates or units so
1396
+ first paint and collapsed layouts do not throw. Invalid custom root sizes and
1397
+ `referencePixelsPerUnit` still throw because they are configuration errors.
1398
+ - Use Three.js `localToWorld`, `worldToLocal`, `Vector3.project/unproject`,
1399
+ `set`, `toArray`, and `fromArray` directly; do not add SDK wrappers for them.
1400
+
1401
+ ### Scene-object store
1402
+
1403
+ ```ts
1404
+ import {
1405
+ addComponentToObject,
1406
+ createDefaultComponent,
1407
+ createSceneObject,
1408
+ deleteSceneObject,
1409
+ getOrderedComponents,
1410
+ moveComponent,
1411
+ removeComponent,
1412
+ type SceneObjectStore,
1413
+ } from "@vincentt-xr/sdk/scene-object";
1414
+
1415
+ const cube = createSceneObject({ name: "Cube" });
1416
+ const transform = createDefaultComponent("Transform", cube.id);
1417
+ const mesh = createDefaultComponent("MeshRenderer", cube.id);
1418
+ cube.componentIds = [transform.id, mesh.id];
1419
+
1420
+ const store: SceneObjectStore = {
1421
+ objects: { [cube.id]: cube },
1422
+ components: { [transform.id]: transform, [mesh.id]: mesh },
1423
+ };
1424
+
1425
+ const ordered = getOrderedComponents(store, cube.id);
1426
+ ```
1427
+
1428
+ `SceneObject` owns identity, display name, enabled state, optional `renderGroupId`,
1429
+ optional hierarchy fields, and ordered `componentIds`. `SceneComponent` owns one
1430
+ typed settings payload and points back to its object. `ComponentRegistry` maps
1431
+ component types to those settings.
1432
+
1433
+ Persisted scenes may still provide the deprecated `layerId`; migration treats it
1434
+ as a fallback when `renderGroupId` is absent. New code should only write
1435
+ `renderGroupId`.
1436
+
1437
+ Nested scene-object fields:
1438
+
1439
+ - `parentObjectId?: SceneObjectId | null` points to the parent object.
1440
+ - `childObjectIds?: SceneObjectId[]` stores ordered child IDs for hierarchy and
1441
+ runtime recursion.
1442
+
1443
+ Nested transforms are evaluated through the runtime object tree. Child objects
1444
+ render inside the parent object, so parent movement affects descendants while
1445
+ each child keeps its own local component settings.
1446
+
1447
+ Store helpers are immutable and preserve extra fields on an app-extended store:
1448
+
1449
+ - `addComponentToObject(store, component)` adds the component and its ID.
1450
+ - `moveComponent(store, objectId, componentId, nextIndex)` reorders one object.
1451
+ - `removeComponent(...)` removes the component and its ID.
1452
+ - `deleteSceneObject(store, objectId)` removes the object and all owned components.
1453
+ - `replaceSceneComponent(store, component)` replaces one component.
1454
+ - `replaceSceneObject(store, object)` replaces one object.
1455
+ - `setSceneObjectVisible(store, objectId, visible)` toggles object `enabled`.
1456
+ - `getSceneObjectChildren(store, objectId)` resolves explicit and implicit child
1457
+ objects in hierarchy order.
1458
+ - `setSceneObjectParent(store, objectId, parentObjectId, nextIndex?)` updates both
1459
+ sides of a parent/child relationship and rejects cycles or mixed transform
1460
+ domains. Parent `Transform` under `Transform`, and `ScreenTransform` under
1461
+ `ScreenTransform`. Existing child values become local-to-parent values.
1462
+ - `setSceneObjectParentPreserveWorldTransform(store, objectId, parentObjectId,
1463
+ nextIndex?)` is the editor seam for drag/drop and parent dropdowns. It updates
1464
+ the hierarchy and recalculates the child's local `Transform` or
1465
+ `ScreenTransform` so the object does not visually jump when nested or
1466
+ un-nested. This is exact for representable transforms. Rotation combined with
1467
+ non-uniform ancestor scale introduces shear, which neither component stores,
1468
+ so preservation in that case is approximate.
1469
+ - `createOrderedComponentsSelector()` creates a memoized ordered selector.
1470
+
1471
+ ### `<SceneObjectRenderer>`
1472
+
1473
+ `SceneObjectRenderer` reads an object plus its ordered components and renders the
1474
+ supported composition. Import the renderer from core and its store type from
1475
+ `/scene-object`.
1476
+
1477
+ ```tsx
1478
+ import { SceneObjectRenderer } from "@vincentt-xr/sdk";
1479
+ import type { SceneComponent, SceneObjectStore } from "@vincentt-xr/sdk/scene-object";
1480
+
1481
+ <SceneObjectRenderer
1482
+ objectId={cube.id}
1483
+ store={store}
1484
+ editorMode
1485
+ onComponentChange={(component: SceneComponent) => replaceComponent(component)}
1486
+ />;
1487
+ ```
1488
+
1489
+ Props are `objectId`, `store`, optional `editorMode`, optional
1490
+ `onComponentChange`, and optional `fallback`. Supported compositions are:
1491
+
1492
+ - `Transform` + `MeshRenderer`
1493
+ - `Transform` + `Light`
1494
+ - `Transform` + `MeshRenderer` with optional `LookAt` and `PinToMesh`
1495
+ - `Transform` only as an invisible 3D null/container
1496
+ - `ScreenTransform` only as an invisible 2D null/container
1497
+ - `ScreenTransform` + `ScreenImage`
1498
+ - `ScreenTransform` + `ScreenText`
1499
+
1500
+ For mesh objects, `SceneObjectRenderer` registers the object id at runtime so
1501
+ behavior components can resolve serialized target ids. Inspector/editor changes
1502
+ must route through `onComponentChange` and replace the selected object's
1503
+ component in the store; do not keep separate page-local copies of transform,
1504
+ material, light, LookAt, or PinToMesh state.
1505
+
1506
+ In editor mode, transform changes return through `onComponentChange`.
1507
+ Unsupported compositions render nothing in production and use `fallback` in
1508
+ editor mode.
1509
+
1510
+ ---
1511
+
1512
+ ## `<ScreenText>` — Standard screen-space text
1513
+
1514
+ Import from `@vincentt-xr/sdk`. Use this when you want the standard
1515
+ screen-text lane instead of wiring `ScreenTransform` + `TextLabel` yourself.
1516
+ For future Vincentt scenes, prefer `ScreenText` over raw `TextLabel` for end-user
1517
+ screen copy such as prompts, captions, banners, badges, and titles.
1518
+
1519
+ `ScreenText` composes:
1520
+
1521
+ - full-width screen-space lane presets
1522
+ - `verticalAlign: "top" | "center" | "bottom"`
1523
+ - `TextLabel` rendering props such as `textAlign`, overflow, background, stroke, and shadow
1524
+
1525
+ ```tsx
1526
+ import { ScreenSpaceUI, ScreenText } from "@vincentt-xr/sdk";
1527
+
1528
+ <ScreenSpaceUI>
1529
+ <ScreenText
1530
+ name="headline"
1531
+ text="Head-Tilt Trivia"
1532
+ coordinateSpace="canvas"
1533
+ canvasSize={{ width: 720, height: 1280 }}
1534
+ position={[0, 500]}
1535
+ size={[620, 140]}
1536
+ textAlign="center"
1537
+ fontSize={270}
1538
+ bgFit="text"
1539
+ bgColor="transparent"
1540
+ />
1541
+ </ScreenSpaceUI>;
1542
+ ```
1543
+
1544
+ `ScreenText` supports the same screen coordinate contract as `ScreenImage`:
1545
+ `coordinateSpace="normalized"` uses normalized screen units, while
1546
+ `coordinateSpace="canvas"` uses Effect House-style design-canvas pixels
1547
+ (`720x1280` by default, center origin, +y up). Its `size` is the text bounding
1548
+ rectangle. The text renderer then aligns, shrinks, clips, or backgrounds the
1549
+ text inside that rectangle.
1550
+
1551
+ Use `ScreenText` when:
1552
+
1553
+ - you want top / center / bottom screen placement
1554
+ - you want wide-lane text alignment without hand-authoring anchors
1555
+ - you want the default end-user screen text behavior
1556
+
1557
+ Use `TextLabel` when:
1558
+
1559
+ - you need raw text rendering outside the screen-text presets
1560
+ - you want to place text manually in world space or inside a custom transform
1561
+ - you are building a lower-level composite where another component owns layout
1562
+
1563
+ `SceneObjectRenderer` owns the required `ScreenSpaceUI` container when it renders
1564
+ an object composed from `ScreenTransform` + `ScreenText` or `ScreenImage`. Do not
1565
+ wrap those renderer instances in an additional `ScreenSpaceUI`; direct uses of
1566
+ `ScreenText` and `ScreenImage` still require the container shown above.
1567
+
1568
+ Debug panels are independent, selection-driven surfaces. Use `HierarchyPanel` for a
1569
+ render-group-aware scene-object list and render one
1570
+ `ObjectInspectorPanel` for the selected object. Both accept `initialPosition` so the
1571
+ hierarchy and Inspector can be presented side by side instead of overlapping.
1572
+ `ScreenTextSettingsPanel` and `ScreenImageSettingsPanel` already own their complete
1573
+ Inspector UI; pass their optional `inspector` binding when they edit an object from
1574
+ a shared scene store. Do not nest either one inside another `ObjectInspectorPanel`.
1575
+ `ObjectInspectorPanel` is uncontrolled by default, but multi-object editors can pass
1576
+ `position` and `onPositionChange` to preserve one Inspector location while swapping
1577
+ the selected object's panel. `SceneObjectRenderer` forwards screen transform-guide
1578
+ drags through `onComponentChange`; selection policies such as closing active transform
1579
+ controls belong to the parent editor, not to `HierarchyPanel` or the standalone panels.
1580
+
1581
+ ### Reusable XR scene editor debug UI
1582
+
1583
+ For temporary client-app scene setup, prefer the reusable scene editor adapters
1584
+ from `@vincentt-xr/sdk/debug-ui` instead of rewriting viewport, hierarchy, and
1585
+ inspector wiring in each app. These are debug/authoring adapters over the
1586
+ scene-object runtime, not production runtime UI.
1587
+
1588
+ ```tsx
1589
+ import {
1590
+ SceneHierarchyPanel,
1591
+ SceneInspectorPanel,
1592
+ ViewportToolbar,
1593
+ XRSceneEditorProvider,
1594
+ XRSceneViewport,
1595
+ } from "@vincentt-xr/sdk/debug-ui";
1596
+ import {
1597
+ replaceSceneComponent,
1598
+ replaceSceneObject,
1599
+ type RenderGroupSceneObjectStore,
1600
+ } from "@vincentt-xr/sdk/scene-object";
1601
+
1602
+ <XRSceneEditorProvider
1603
+ store={store}
1604
+ selectedObjectId={selectedObjectId}
1605
+ onSelectObject={setSelectedObjectId}
1606
+ onStoreChange={setStore}
1607
+ onComponentChange={(component) =>
1608
+ setStore((current) => replaceSceneComponent(current, component))
1609
+ }
1610
+ onObjectChange={(object) => setStore((current) => replaceSceneObject(current, object))}
1611
+ >
1612
+ <XRSceneViewport />
1613
+ <ViewportToolbar />
1614
+ <SceneHierarchyPanel />
1615
+ <SceneInspectorPanel />
1616
+ </XRSceneEditorProvider>;
1617
+ ```
1618
+
1619
+ All pieces are optional and composable:
1620
+
1621
+ - `XRSceneViewport` renders `XRScene`, an editor-owned camera-texture plane,
1622
+ editor grid, camera, orbit controls, and `SceneObjectRenderer` entries from
1623
+ `createRenderPlan`. The editor video plane consumes the same camera texture as
1624
+ `VideoBackground`, but it is a separate debug-ui plane with `depthTest` and
1625
+ `depthWrite` enabled. Do not add depth props to core `VideoBackground`.
1626
+ - `SceneHierarchyPanel` adapts `HierarchyPanel` to a
1627
+ `RenderGroupSceneObjectStore`.
1628
+ - `SceneInspectorPanel` renders component settings for the selected object and
1629
+ routes edits through `onComponentChange` or `onObjectChange`.
1630
+ - `ViewportToolbar` owns transient viewport controls such as orbit on/off.
1631
+
1632
+ Keep `SceneObjectStore`/`RenderGroupSceneObjectStore` as the only persistent
1633
+ scene data. Selection, orbit state, grid visibility, video-background visibility,
1634
+ and editor camera choices are transient editor session state. Do not keep
1635
+ page-local shadow copies of Transform, MeshRenderer, Light, LookAt, or PinToMesh
1636
+ settings.
1637
+
1638
+ Use these adapters in development/debug surfaces such as `/viewport-editor`
1639
+ or a temporary client-app editor. Omit them for final preview/runtime and render
1640
+ the scene through core runtime components instead.
1641
+
1642
+ For debug pages that need a camera/image/video picker, import
1643
+ `MediaSourceSwitcher` and `useFaceTrackingMediaSourceSession` from
1644
+ `@vincentt-xr/sdk/debug-ui`. `useFaceTrackingMediaSourceSession(activeMediaSource)`
1645
+ applies the selected debug media preset to the current XR session and starts
1646
+ the face-tracking media pipeline used by the debug face examples. The older
1647
+ `useMediaSourceSession(activeMediaSource, { pipeline })` name remains available
1648
+ for custom debug pipelines. Use these only in debug/example UI; production
1649
+ runtime should set the session media source directly.
1650
+
1651
+ ### Debug UI usage in Harness client apps
1652
+
1653
+ #### AI-agent integration contract
1654
+
1655
+ Choose the import door by responsibility. Do not search other doors for an
1656
+ equivalent symbol and do not import SDK source files directly.
1657
+
1658
+ | Agent task | Import door | Symbols |
1659
+ | ----------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------ |
1660
+ | Render managed objects and compose passes | `@vincentt-xr/sdk` | `SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer` |
1661
+ | Guard internal render layers | `@vincentt-xr/sdk` | `MAX_RENDER_GROUP_PASSES`, `getRenderGroupLayer`, `EXTERNAL_BACKGROUND_RENDER_LAYER` |
1662
+ | Create and mutate scene-object state | `@vincentt-xr/sdk/scene-object` | `createSceneObject`, `createDefaultComponent`, store helpers |
1663
+ | Materialize and order render groups | `@vincentt-xr/sdk/scene-object` | `migrateToRenderGroups`, `createRenderPlan`, reorder/move helpers |
1664
+ | Render DOM authoring controls | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`, settings panels |
1665
+ | Type a bulk hierarchy export | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanelCopyConfig` |
1666
+
1667
+ The integrating client, not the SDK panels, must own all mutable editor state:
1668
+
1669
+ - the complete scene store, including `objects`, `components`, `renderGroups`,
1670
+ and `renderGroupOrder`;
1671
+ - `selectedObjectId` and which selected Inspector is rendered;
1672
+ - component replacement and object/group visibility updates;
1673
+ - reorder and compatibility-checked move operations;
1674
+ - the rule that changing selection closes transform controls and guides belonging
1675
+ to the previously selected object; and
1676
+ - the `HierarchyPanelCopyConfig` value.
1677
+
1678
+ Required invariants:
1679
+
1680
+ - Mount DOM debug panels outside `XRScene`; mount R3F/Three.js renderers and
1681
+ controls inside `XRScene`.
1682
+ - Keep both branches below the same parent state and inside `XRProvider`.
1683
+ - Derive group order from `scene.renderGroupOrder`; never hardcode a canonical
1684
+ order such as background/general/tracking/foreground/post.
1685
+ - Use `SequentialRenderPasses` for guaranteed cross-group composition. Do not
1686
+ replace it with a larger Three.js `renderOrder` value.
1687
+ - Treat camera layers as internal visibility masks, not user-facing layer order.
1688
+ - Keep each managed object in exactly one materialized group and exactly once in
1689
+ that group's `objectIds`.
1690
+ - Do not show transform controls for an unselected object. Initial control state
1691
+ must agree with the initial `selectedObjectId`.
1692
+ - Do not create more than `MAX_RENDER_GROUP_PASSES` managed groups.
1693
+ - Treat Copy Config as a versioned export/debug payload, not as an import or
1694
+ persistence API.
1695
+
1696
+ Use these state transitions:
1697
+
1698
+ | Event | Required parent-state update |
1699
+ | ----------------- | -------------------------------------------------------------------------------- |
1700
+ | Select object | Set `selectedObjectId`; disable all existing 2D guides and 3D transform controls |
1701
+ | Edit component | Replace that component immutably in `scene.components` |
1702
+ | Reorder group | Call `reorderRenderGroup`; keep object lists unchanged |
1703
+ | Reorder object | Call `reorderObjectWithinGroup`; change only the selected group's `objectIds` |
1704
+ | Move object | Validate compatibility, then call `moveObjectToGroup` |
1705
+ | Toggle visibility | Update the object or group; do not encode visibility in selection state |
1706
+ | Render frame | Derive `createRenderPlan(scene)` and active pass layers from current state |
1707
+
1708
+ Failure recovery for generated integrations:
1709
+
1710
+ | Symptom | Likely cause | Required fix | Do not |
1711
+ | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- |
1712
+ | “R3F hooks can only be used within Canvas” | R3F renderer/control mounted in the DOM branch, or duplicate React/R3F packages | Move the scene component inside `XRScene`; in link mode pin React, React DOM, Three.js, R3F, and Drei to client copies | Move DOM panels into `XRScene` |
1713
+ | Hierarchy order changes but visual composition does not | Only `renderOrder` or array order changed | Recompute the render plan and pass the ordered layers to `SequentialRenderPasses` | Expose camera-layer numbers as visual order |
1714
+ | Wrong object's gizmo remains visible | Selection changed without clearing component control flags | Disable all stale transform controls/guides in the same selection transition | Assume `HierarchyPanel` owns control state |
1715
+ | Screen transform drag stops after one move | An unstable callback caused active pointer listeners to be replaced | Use current SDK components and keep component replacement callbacks wired to parent state | Fork private drag logic into the DOM panel |
1716
+ | Cross-group drop silently fails | Target policy rejects the object's component composition | Call `validateGroupCompatibility` and preserve the original store on `false` | Force incompatible membership |
1717
+ | Group is omitted from the render plan | Its zero-based index is `>= MAX_RENDER_GROUP_PASSES` (`30`), so the 31st group onward is beyond the managed-pass limit | Prevent creation using `MAX_RENDER_GROUP_PASSES` | Reuse/clamp multiple groups onto one layer |
1718
+ | Copied JSON cannot be restored | Copy Config has no public importer | Treat it as export/debug handoff only | Invent an undocumented restore API |
1719
+
1720
+ Harness clients must keep the DOM authoring UI and R3F scene integration in sibling
1721
+ branches under the same parent-owned state:
1722
+
1723
+ ```text
1724
+ XRProvider
1725
+ └─ AspectRatioContainer
1726
+ ├─ XRScene (R3F Canvas)
1727
+ │ ├─ camera, lights, and scene content
1728
+ │ ├─ SequentialRenderPasses
1729
+ │ ├─ SceneObjectRenderer instances
1730
+ │ └─ transform guides and controls
1731
+ ├─ HierarchyPanel
1732
+ └─ selected-object Inspector or standalone settings panel
1733
+ ```
1734
+
1735
+ Mount `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`,
1736
+ `ScreenTextSettingsPanel`, and `ScreenImageSettingsPanel` outside `XRScene`.
1737
+ They render DOM portals and communicate through props and callbacks. Mount
1738
+ `SceneObjectRenderer`, `SequentialRenderPasses`, `ScreenSpaceUI`, `Transform3D`,
1739
+ and all Three.js content inside `XRScene` because they use R3F Canvas context.
1740
+ Keep both branches inside `XRProvider`.
1741
+
1742
+ The parent Harness component owns:
1743
+
1744
+ - the complete scene-object store;
1745
+ - `selectedObjectId`;
1746
+ - component replacement;
1747
+ - group and object reorder callbacks;
1748
+ - visibility callbacks;
1749
+ - compatibility-checked moves; and
1750
+ - the bulk Copy Config value.
1751
+
1752
+ The data loop is:
1753
+
1754
+ ```text
1755
+ Hierarchy / Inspector callback
1756
+ ↓
1757
+ parent scene store
1758
+ ↓
1759
+ selection + render plan + active pass layers
1760
+ ↓
1761
+ SceneObjectRenderer inside XRScene
1762
+ ↓
1763
+ transform-control change
1764
+ └──────────────→ parent scene store
1765
+ ```
1766
+
1767
+ If React reports that R3F hooks can only be used within Canvas, a scene renderer,
1768
+ render-pass controller, transform guide, or Three.js control has crossed into the
1769
+ DOM panel branch. Do not solve this by moving the panels into `XRScene`; move the
1770
+ R3F-dependent scene component back inside `XRScene` and keep the shared state above
1771
+ both branches.
1772
+
1773
+ The working `test001` Harness client is the canonical external integration. It runs
1774
+ a Cube, Screen Image, and Screen Text together with `HierarchyPanel`,
1775
+ selection-driven Inspectors, component editors, group/object ordering, visibility,
1776
+ transform controls, sequential render passes, and bulk Copy Config. The SDK
1777
+ `/layer-panel-test` page is the matching in-repo fixture.
1778
+
1779
+ For SDK co-development, run `pnpm build:watch` in `vincentt-xr-sdk` and
1780
+ `SDK_LINK=1 pnpm dev` in the Harness client. Use a different `PORT` when the SDK
1781
+ example is already using 5173, for example
1782
+ `PORT=5174 SDK_LINK=1 pnpm dev`. Also validate the client typecheck and linked
1783
+ production build; the in-repo SDK example alone does not validate package exports
1784
+ or Canvas placement in a separate client.
1785
+
1786
+ ### Render groups and hierarchy ordering
1787
+
1788
+ Import render-group types, policies, migration, compatibility, reorder helpers, and
1789
+ `createRenderPlan` from `@vincentt-xr/sdk/scene-object`. Import
1790
+ `SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer`,
1791
+ `getRenderGroupLayer`, and `EXTERNAL_BACKGROUND_RENDER_LAYER` from core.
1792
+
1793
+ Use `migrateToRenderGroups(store, objectOrder)` to group a flat scene-object store.
1794
+ It preserves compatible `renderGroupId` assignments, infers missing or incompatible
1795
+ assignments from components, and creates only the groups required by actual scene
1796
+ objects. Their initial order follows the first occurrence in `objectOrder`; there is
1797
+ no fixed render-order list. New
1798
+ objects can call `getDefaultRenderGroupId(components)` after their components are
1799
+ assembled.
1800
+
1801
+ The ownership boundary is:
1802
+
1803
+ | Concern | Owner |
1804
+ | -------------------------------------------------- | -------------------------- |
1805
+ | Object identity, display name, and component order | `SceneObject` |
1806
+ | Object behavior and settings | `SceneComponent` |
1807
+ | Cross-domain composition and group visibility | `RenderGroup` |
1808
+ | Order inside a group | `RenderGroup.objectIds` |
1809
+ | Order between groups | `renderGroupOrder` |
1810
+ | Structural commands | `HierarchyPanel` callbacks |
1811
+ | Property editing and selection policy | Parent editor + Inspector |
1812
+ | Object-to-pass mapping | `createRenderPlan()` |
1813
+ | Final frame composition | `SequentialRenderPasses` |
1814
+
1815
+ The Hierarchy and Inspector emit commands but do not own scene state or rendering.
1816
+ Every managed object must belong to exactly one materialized group, appear exactly
1817
+ once across all group `objectIds` arrays, and have a matching `renderGroupId`.
1818
+ Selection is editor state and never changes render order.
1819
+
1820
+ Use only the immutable helpers to change structure:
1821
+
1822
+ - `reorderRenderGroup(store, groupId, nextIndex)` changes group composition order.
1823
+ - `reorderObjectWithinGroup(store, groupId, objectId, nextIndex)` sorts within a group.
1824
+ - `moveObjectToGroup(...)` moves only when `validateGroupCompatibility(...)` passes.
1825
+ - `createRenderPlan(store)` returns top-first entries and stable 1000-value sorting bands.
1826
+
1827
+ Unknown group/object IDs, incompatible targets, and reorders to the current index
1828
+ are safe no-ops that return the original store. `validateGroupCompatibility` is
1829
+ read-only and returns `false` for missing objects, missing groups, or rejected
1830
+ component compositions.
1831
+
1832
+ Pass each plan entry's `renderOrder` and `renderLayer` to `SceneObjectRenderer`.
1833
+ Mount one `SequentialRenderPasses` with the active plan layers in top-first order.
1834
+ It takes over the R3F loop, preserves color between groups, and clears depth before
1835
+ each later group. Bottom groups therefore composite in front even across Three.js's
1836
+ opaque and transparent queues. `captureFrame()` uses the same grouped renderer.
1837
+
1838
+ These mechanisms are different:
1839
+
1840
+ - `renderGroupOrder` controls cross-group sequential composition. The first group
1841
+ renders first; the last group renders last and appears in front.
1842
+ - `RenderGroup.objectIds` controls order inside one group.
1843
+ - Three.js camera layers are internal visibility masks; their numeric values do not
1844
+ define visual order.
1845
+ - `renderOrder` is a sorting value inside one render call. It cannot by itself
1846
+ impose ordering across opaque and transparent queues.
1847
+
1848
+ ```tsx
1849
+ const plan = createRenderPlan(scene);
1850
+ const passLayers = scene.renderGroupOrder.flatMap((groupId, groupIndex) =>
1851
+ scene.renderGroups[groupId]?.enabled && scene.renderGroups[groupId].objectIds.length
1852
+ ? [getRenderGroupLayer(groupIndex)]
1853
+ : [],
1854
+ );
1855
+
1856
+ <SequentialRenderPasses layers={passLayers} />;
1857
+ {
1858
+ plan.map((entry) => (
1859
+ <SceneObjectRenderer
1860
+ key={entry.objectId}
1861
+ objectId={entry.objectId}
1862
+ store={scene}
1863
+ renderOrder={entry.renderOrder}
1864
+ renderLayer={entry.renderLayer}
1865
+ />
1866
+ ));
1867
+ }
1868
+ ```
1869
+
1870
+ Layer 0 is shared across passes for lights and non-renderable scene infrastructure.
1871
+ All visible managed project content in a grouped scene must be assigned to a group
1872
+ layer. `SequentialRenderPasses` also enables its layers on the R3F interaction
1873
+ raycaster. `RenderGroupLayer` leaves Drei `TransformControls` on shared layer 0,
1874
+ so 2D guides and 3D move/rotate/scale gizmos remain draggable after render-group
1875
+ isolation.
1876
+
1877
+ For legacy flat camera backgrounds, render `VideoBackground` outside managed
1878
+ scene-object content. For the reusable scene editor, use `XRSceneViewport`; it
1879
+ owns a separate depth-writing camera-texture plane for editor-only 3D placement.
1880
+
1881
+ `HierarchyPanel` receives ordered `groups`, scene `objects`, selection callbacks,
1882
+ and separate callbacks for group reorder, within-group reorder, and explicit moves.
1883
+ It owns collapse UI and drop feedback, but the parent store owns selection, group
1884
+ visibility, compatibility validation, the Inspector, and rendering.
1885
+ Empty reserved groups are not materialized in the store or presented in the hierarchy.
1886
+ The panel is expanded by default and can fold to its title button without losing
1887
+ its position, current selection, or per-group collapse state. The normal hierarchy
1888
+ shows human-readable names only; internal domains, IDs, camera layers, and ordering
1889
+ values belong in deliberate debug tooling rather than the primary object list.
1890
+ Pass a `HierarchyPanelCopyConfig` value as `copyConfig` to show one bulk
1891
+ **Copy Config** action. The panel writes that value as formatted JSON and confirms
1892
+ with **Copied Config**. For a complete scene
1893
+ handoff, include `renderGroupOrder`, `renderGroups`, `objects`, and `components`;
1894
+ omit transient editor state such as current selection and panel position. The copied
1895
+ group order therefore contains only groups backed by current scene objects.
1896
+
1897
+ Use this versioned shape for a complete handoff:
1898
+
1899
+ ```ts
1900
+ type SceneConfigV1 = {
1901
+ kind: "scene";
1902
+ version: 1;
1903
+ renderGroupOrder: RenderGroupId[];
1904
+ renderGroups: Record<RenderGroupId, RenderGroup>;
1905
+ objects: Record<SceneObjectId, SceneObject>;
1906
+ components: Record<SceneComponentId, SceneComponent>;
1907
+ };
1908
+ ```
1909
+
1910
+ This is currently an export/debugging contract, not a guaranteed round-trip
1911
+ format: there is no public scene-config importer. Exclude selection, panel
1912
+ position, fold/collapse state, active drag state, feedback messages, and any
1913
+ flat external `VideoBackground`.
1914
+
1915
+ Current compatibility and limits:
1916
+
1917
+ | Domain | Required composition | Status |
1918
+ | ------------------ | -------------------------------------------------- | --------- |
1919
+ | General / 3D Scene | `Transform` + `MeshRenderer` | supported |
1920
+ | 2D Foreground | `ScreenTransform` + `ScreenText` or `ScreenImage` | supported |
1921
+ | Background | editor camera-texture plane or external background | partial |
1922
+ | AR Tracking | tracking scene-object components | reserved |
1923
+ | Post Effects | post-processing scene object | reserved |
1924
+
1925
+ Reserved groups are not materialized or displayed until compatible managed scene
1926
+ objects exist. Each active group adds one scene render, with a maximum of
1927
+ `MAX_RENDER_GROUP_PASSES` (currently 30) managed passes. Import that constant from
1928
+ `@vincentt-xr/sdk` when guarding group creation. `createRenderPlan` omits groups
1929
+ beyond the limit instead of throwing or assigning colliding camera layers.
1930
+ `EXTERNAL_BACKGROUND_RENDER_LAYER` remains temporary infrastructure for legacy
1931
+ external background composition. The reusable scene editor should prefer its
1932
+ own camera-texture plane instead of treating `VideoBackground` as a managed
1933
+ scene object.
1934
+
1935
+ An explicit `renderGroupId` assignment is retained only when its policy accepts the
1936
+ object's components; otherwise grouping uses the inferred compatible domain. Object
1937
+ IDs, component IDs, and component settings remain unchanged.
1938
+
1939
+ The render-group tests protect derived group order, compatible defaults, local
1940
+ object reorder, rejected move no-ops, stable render bands/layers, sequential depth
1941
+ clearing, grouped capture, and interaction across active camera layers. The
1942
+ compiling reference implementation is `/layer-panel-test` in the SDK example.
1943
+
1944
+ ### Shared debug authoring surface for `ScreenText`
1945
+
1946
+ The SDK also ships a dev-only inspector at `lib/debug-ui/screen-text-settings`
1947
+ for `ScreenText` authoring. It groups the text renderer, transform, style, and
1948
+ background controls into separate sections so creators can inspect the lane
1949
+ without shipping the debug chrome in production.
1950
+
1951
+ The panel's transform copy button copies a JSON payload for the selected object
1952
+ including `name`, `kind: "screenText"`, `transform`, `content`, `style`,
1953
+ `layout`, and `screenSpaceUI`. The `transform` includes `coordinateSpace` and
1954
+ `canvasSize`. Use that payload as the handoff from human layout tuning back to
1955
+ code.
1956
+
1957
+ ### Legacy debug layer panel
1958
+
1959
+ The SDK ships a reusable `LayerPanel` in `lib/debug-ui/layer-panel` for
1960
+ flat-stack compatibility. New grouped editors should use `HierarchyPanel`.
1961
+ Both are intended for dev / creator tooling only. Production shipping surfaces
1962
+ should not include either panel.
1963
+
1964
+ ---
1965
+
376
1966
  ## `<TextLabel>` — Canvas-rendered text panel
377
1967
 
378
1968
  Import from `@vincentt-xr/sdk`. Billboarded in 3D space.
379
1969
 
380
- | prop | type | default | notes |
381
- | -------------- | ------------------- | ---------------------- | --------------------------------------------------------------------------------------------- |
382
- | `text` | `string` | **required** | The text to render. |
383
- | `color` | `string` | `#ffffff` | Text color. |
384
- | `fontSize` | `number` | `48` | Canvas px at the default 512px canvas — NOT world units. Chips/badges typically need 200-320. |
385
- | `bgColor` | `string` | `rgba(20,40,110,0.88)` | Panel background; use `"transparent"` for text-only. |
386
- | `borderColor` | `string` | — | Border color; omit for no border. |
387
- | `borderRadius` | `number` | `24` | Corner radius; 0 for square. |
388
- | `aspect` | `number` | `2.5` | Panel shape (width ÷ height). >1 = wide/banner, <1 = tall. |
389
- | `scale` | `[x,y,z] \| number` | `1` | World size. Combine with `aspect` to make wide banners. |
390
- | `position` | `[x,y,z]` | `[0,0,0]` | World position. +Y up, -Y down. |
391
- | `name` | `string` | — | Set on every element you create. |
1970
+ | prop | type | default | notes |
1971
+ | --------------- | ------------------------------------------------- | ---------------------- | --------------------------------------------------------------------------------------------- |
1972
+ | `text` | `string` | **required** | The text to render. |
1973
+ | `lines` | `string[]` | — | Optional precomputed lines. Skips automatic word wrapping. |
1974
+ | `color` | `string` | `#ffffff` | Text color. |
1975
+ | `strokeLayers` | `{ color, width, opacity? }[]` | — | Optional multi-stroke stack, drawn outer-to-inner. |
1976
+ | `strokeColor` | `string` | — | Optional text stroke color. Pair with `strokeWidth`. |
1977
+ | `strokeWidth` | `number` | `0` | Text stroke thickness in canvas px. |
1978
+ | `shadowLayers` | `{ color, blur, offsetX?, offsetY?, opacity? }[]` | — | Optional multi-shadow/glow stack, drawn back-to-front. |
1979
+ | `shadowColor` | `string` | — | Optional text shadow / glow color. |
1980
+ | `shadowBlur` | `number` | `0` | Shadow blur radius in canvas px. |
1981
+ | `shadowOffsetX` | `number` | `0` | Shadow X offset in canvas px. |
1982
+ | `shadowOffsetY` | `number` | `0` | Shadow Y offset in canvas px. |
1983
+ | `fontSize` | `number` | `48` | Canvas px at the default 512px canvas — NOT world units. Chips/badges typically need 200-320. |
1984
+ | `textAlign` | `"left" \| "center" \| "right"` | `"center"` | Horizontal text alignment inside the available lane. |
1985
+ | `overflow` | `"clip" \| "clamp" \| "ellipsis" \| "shrink"` | — | Overflow behavior. `clamp` keeps full visible lines, `clip` allows canvas-edge clipping. |
1986
+ | `resizeToFit` | `boolean` | — | Keep shrinking until the text fits or `minFontSize` is reached. |
1987
+ | `minFontSize` | `number` | `12` | Minimum shrink target in canvas px. |
1988
+ | `bgColor` | `string` | `rgba(20,40,110,0.88)` | Panel background; use `"transparent"` for text-only. |
1989
+ | `bgFit` | `"full" \| "text"` | `"full"` | Background fit mode. Use `"text"` for pill/capsule backgrounds that hug the text block. |
1990
+ | `borderColor` | `string` | — | Border color; omit for no border. |
1991
+ | `borderRadius` | `number` | `24` | Corner radius; 0 for square. |
1992
+ | `aspect` | `number` | `2.5` | Panel shape (width ÷ height). >1 = wide/banner, <1 = tall. |
1993
+ | `scale` | `[x,y,z] \| number` | `1` | World size. Combine with `aspect` to make wide banners. |
1994
+ | `position` | `[x,y,z]` | `[0,0,0]` | World position. +Y up, -Y down. |
1995
+ | `name` | `string` | — | Set on every element you create. |
392
1996
 
393
1997
  **Gotchas:**
394
1998
 
395
1999
  - `fontSize` is canvas px, not world units. Chips/badges need 200-320, not 16-48.
396
2000
  - `fontSize` does not change panel shape — use `aspect` (shape) and `scale` (size).
397
- - `TextLabel` ignores the `visible` prop — wrap in `<group visible={false}>` to start hidden.
2001
+ - `textAlign` affects both glyph placement and `bgFit="text"` background placement.
2002
+ - `TextLabel` respects the `visible` prop.
2003
+ - For text-first 2D treatments, prefer `strokeColor` + `strokeWidth` and `shadowColor` + `shadowBlur` before stacking multiple `TextLabel`s. Native text effects keep the glyph edges aligned and resize cleanly.
2004
+ - Use `strokeLayers` / `shadowLayers` when you need richer looks like sticker text, double-outline, or offset-drop-shadow. The single `stroke*` / `shadow*` props are shorthand for one layer and can coexist with the arrays.
2005
+ - `TextLabel` now owns real overflow behavior. If you need exact authored line breaks, pass `text` with `\n` or provide `lines` directly; those lines will be rendered as the source layout instead of being re-wrapped from one flattened string.
2006
+ - Overflow semantics:
2007
+ - `clip`: draw all resolved lines and let the canvas bounds crop them, including partial last lines.
2008
+ - `clamp`: keep only full visible lines within the line budget.
2009
+ - `ellipsis`: same as clamp, but ellipsize the last visible line.
2010
+ - `shrink`: reduce font size until the content fits or `minFontSize` is reached, then clamp if it still overflows.
398
2011
  - **Inside a `<ScreenTransform>`, the parent stretches the label to fill the anchor rect — `aspect` must equal the rect's aspect or the banner distorts and leaves dead space around the text.** The rect's aspect is `(right − left) / (top − bottom)`. So for a full-width top banner `anchors={{ left: -1, right: 1, top: 1, bottom: 0.76 }}` → aspect `= 2 / 0.24 ≈ 8.3`. Set `borderRadius={0}` for a true edge-to-edge banner. (The rect controls size + placement — don't add `scale`/`position` to fill it; set the anchors instead.)
2012
+ - If you want the SDK to own that anchor math, use `<ScreenText>` instead of pairing `<ScreenTransform>` with `<TextLabel>` manually.
399
2013
 
400
2014
  ```tsx
401
2015
  // Full-width banner pinned to the top, INSIDE a ScreenTransform.
@@ -445,21 +2059,28 @@ import { AspectRatioContainer } from "@vincentt-xr/sdk";
445
2059
 
446
2060
  Grab a photo of the live render or record the canvas, then save or share it.
447
2061
 
448
- ### `session.captureFrame()` + `dataURLtoFile()` — Single-shot photo
2062
+ ### `useFrameCapture()` + `dataURLtoFile()` — Single-shot photo
449
2063
 
450
- `captureFrame()` forces a fresh render and returns a `data:` URL of the current
451
- frame. `dataURLtoFile(dataUrl, filename)` turns it into a `File` for upload or the
452
- Web Share API.
2064
+ Use `useFrameCapture` for UI/effect logic. It exposes `status`, `capturePhoto`,
2065
+ `clear`, `dataUrl`, and `error`, plus `onCaptureStart`, `onCaptureComplete`, and
2066
+ `onError`. Status is `idle | capturing | completed | error`. The underlying
2067
+ capture forces a fresh render and returns a `data:` URL; `dataURLtoFile` turns
2068
+ it into a `File` for upload or the Web Share API. `capturePhoto()` resolves to
2069
+ the URL on success or `null` after setting `status: "error"` and `error` on
2070
+ failure; callers do not need a rejection handler.
453
2071
 
454
2072
  ```tsx
455
- import { dataURLtoFile } from "@vincentt-xr/sdk";
456
- import { useXRContext } from "@vincentt-xr/sdk/low-level";
2073
+ import { dataURLtoFile, useFrameCapture } from "@vincentt-xr/sdk";
457
2074
 
458
- const { session } = useXRContext();
459
- const dataUrl = session.captureFrame();
460
- const file = dataURLtoFile(dataUrl, "photo.png");
2075
+ const photo = useFrameCapture({
2076
+ onCaptureComplete: (dataUrl) => share(dataURLtoFile(dataUrl, "photo.png")),
2077
+ });
2078
+ await photo.capturePhoto();
461
2079
  ```
462
2080
 
2081
+ For synchronous low-level code, `session.captureFrame()` remains available via
2082
+ `useXRContext` from `@vincentt-xr/sdk/low-level`.
2083
+
463
2084
  ### `useMediaRecorder()` — Record the canvas to a video Blob
464
2085
 
465
2086
  Records a `MediaStream` (from the canvas's `captureStream()`) to a `Blob`. Picks
@@ -476,6 +2097,20 @@ await rec.start(canvas.captureStream(30));
476
2097
  rec.stop(); // -> rec.recordedBlob is the video Blob
477
2098
  ```
478
2099
 
2100
+ Use `rec.status` as the source of truth:
2101
+ `idle | starting | recording | paused | stopping | completed | error`.
2102
+ Controls are `start`, `pause`, `resume`, `stop`, and `clear`. Events are
2103
+ `onRecordingStart`, `onRecordingPause`, `onRecordingResume`,
2104
+ `onRecordingStop`, `onRecordingComplete`, and `onError`. Keep `isRecording`
2105
+ and `isPaused` only as convenience booleans; do not invent parallel state.
2106
+ When `passThroughStream` is configured, call `start()` without an argument;
2107
+ otherwise call `start(stream)` with the canvas capture stream.
2108
+ The verified `/capture-recording` example exercises the complete photo and video
2109
+ lifecycle against the live XR canvas. It displays a recording HUD and elapsed
2110
+ time for `starting`, `recording`, `paused`, `stopping`, and `completed`, then
2111
+ previews the resulting `recordedBlob` in a video player. Photo and recording
2112
+ flows were verified with a real webcam on 26 July 2026.
2113
+
479
2114
  ---
480
2115
 
481
2116
  ## Audio
@@ -535,3 +2170,154 @@ function SoundToggle() {
535
2170
  Audio assets: m4a (AAC). A rare app needing a second, independent mixer can
536
2171
  `new AudioEngine()` and drive it directly; the free functions always target the
537
2172
  one default engine.
2173
+
2174
+ ## 3D behavior components: LookAt and PinToMesh
2175
+
2176
+ `LookAt` and `PinToMesh` are partial-stage XRScene behavior components. They are
2177
+ exported for editor/API prototyping, but they are not yet a fully verified
2178
+ Effect House parity surface.
2179
+
2180
+ `LookAt` rotates its child toward a target object resolved either from a direct
2181
+ `Object3D` prop or from serialized scene-object target ids. Treat it as suitable
2182
+ for current editor experiments, not final production behavior, until point vs
2183
+ direction mode, axis conventions, offset ordering, and missing-target behavior
2184
+ are covered by tests.
2185
+
2186
+ When `SceneObjectRenderer` renders nested objects, child scene objects do not
2187
+ inherit the parent's LookAt reorientation by default. `LookAtSettings` includes
2188
+ `inheritChildren: boolean` for the special case where children should rigidly
2189
+ follow the parent's look-at facing. Defaults from
2190
+ `createDefaultComponent("LookAt", objectId)` set `inheritChildren: false`.
2191
+
2192
+ `PinToMesh` wraps a source object and pins that source wrapper to a target mesh
2193
+ surface. Pass `PinToMeshSettings` with
2194
+ `enabled: true` and a non-null `targetObjectId`; default settings are disabled
2195
+ with no target. The component supports static mesh surface lookup, UV input,
2196
+ orientation, vertex-normal option, offset space, offset position, and offset
2197
+ rotation. Offset position and offset rotation default to `{ x: 0, y: 0, z: 0 }`.
2198
+ When PinToMesh is disabled or the target becomes unavailable, the runtime pin
2199
+ wrapper resets to local position `0,0,0` and identity rotation so the child
2200
+ `Transform3D` remains the authored source of truth.
2201
+
2202
+ Pin wrapper rotation is identity-based for `orientation: "none"` and
2203
+ `orientation: "position"`. `orientation: "positionAndDirection"` aligns the
2204
+ wrapper to the target surface normal before offset rotation is applied. Offset
2205
+ rotation is reapplied from a clean base each frame and must not compound over
2206
+ time.
2207
+
2208
+ Use `PinToMeshSettingsPanel` from `@vincentt-xr/sdk/debug-ui` for editor
2209
+ controls. The panel uses Transform-style text number inputs for UV and offsets:
2210
+ visible zero defaults, decimal validation, 0.1 arrow-step increments, Shift for
2211
+ larger steps, and Alt for finer steps. Serialized scenes should store stable
2212
+ scene-object ids for targets; runtime renderers resolve those ids to registered
2213
+ `Object3D` instances.
2214
+
2215
+ Direct runtime usage:
2216
+
2217
+ ```tsx
2218
+ import { LookAt, PinToMesh, Transform3D, MeshRenderer } from "@vincentt-xr/sdk";
2219
+
2220
+ <PinToMesh
2221
+ target={targetObject}
2222
+ settings={{
2223
+ enabled: true,
2224
+ targetObjectId: "base-plane",
2225
+ orientation: "positionAndDirection",
2226
+ uv: { x: 0.5, y: 0.5 },
2227
+ vertexNormal: true,
2228
+ offsetSpace: "local",
2229
+ offsetPosition: { x: 0, y: 0.25, z: 0 },
2230
+ offsetRotation: { x: 0, y: 0, z: 0 },
2231
+ }}
2232
+ >
2233
+ <Transform3D value={transformSettings}>
2234
+ <LookAt
2235
+ target={lookTargetObject}
2236
+ settings={{
2237
+ enabled: true,
2238
+ targetObjectId: "target-object",
2239
+ mode: "point",
2240
+ offsetRotation: { x: 0, y: 0, z: 0 },
2241
+ }}
2242
+ >
2243
+ <MeshRenderer value={meshSettings} />
2244
+ </LookAt>
2245
+ </Transform3D>
2246
+ </PinToMesh>;
2247
+ ```
2248
+
2249
+ Serialized component defaults:
2250
+
2251
+ - `createDefaultComponent("LookAt", objectId)` returns disabled LookAt settings
2252
+ with `targetObjectId: null`, `mode: "point"`, and zero offset rotation.
2253
+ - `createDefaultComponent("PinToMesh", objectId)` returns disabled PinToMesh
2254
+ settings with `targetObjectId: null`, UV `{ x: 0.5, y: 0.5 }`,
2255
+ `orientation: "none"`, `vertexNormal: false`, `offsetSpace: "local"`, and
2256
+ zero position/rotation offsets.
2257
+
2258
+ Current limitations:
2259
+
2260
+ - UV behavior is not final. Missing UVs currently fall back to target bounds
2261
+ center behavior instead of a visible invalid-target state.
2262
+ - LookAt has a minimal API: `enabled`, `targetObjectId`, `mode`, and
2263
+ `offsetRotation`. Axis remapping and world-up controls are still planned.
2264
+ - Runtime target resolution is scoped by `XRProvider` through an internal
2265
+ runtime-object registry. Direct calls to `registerRuntimeObject` and
2266
+ `resolveRuntimeObject` outside an `XRProvider` use a legacy fallback registry
2267
+ and should be avoided for multi-scene pages.
2268
+
2269
+ ## 3D light scene objects
2270
+
2271
+ `LightObject` renders a serialized light component through `Transform3D`.
2272
+ Import from `@vincentt-xr/sdk`.
2273
+
2274
+ ```tsx
2275
+ import { LightObject } from "@vincentt-xr/sdk";
2276
+
2277
+ <LightObject
2278
+ transform={transformSettings}
2279
+ settings={{
2280
+ enabled: true,
2281
+ type: "directional",
2282
+ color: "#ffffff",
2283
+ intensity: 2.2,
2284
+ castShadow: true,
2285
+ distance: 10,
2286
+ angle: Math.PI / 6,
2287
+ penumbra: 0.2,
2288
+ decay: 2,
2289
+ showHelper: false,
2290
+ }}
2291
+ />;
2292
+ ```
2293
+
2294
+ `LightSettings.type` is `"ambient" | "directional" | "point" | "spot"`.
2295
+ Directional, point, and spot lights support `castShadow`; ambient lights only
2296
+ contribute global light. `showHelper` draws a small non-physical editor marker
2297
+ for the light object and defaults to `false`; enable it only in editor/debug
2298
+ surfaces.
2299
+
2300
+ For an Effect House-style declarative player, use `<AudioPlayer>`. It supports
2301
+ `once`, a finite `loop` count, and `infinite` playback, plus autoplay intent,
2302
+ per-player mute, and a clamped 0–100 volume. Browser autoplay policy still
2303
+ applies: audible autoplay may reject until a user gesture, surfaced through
2304
+ `onError`. The component is headless (it renders a hidden `<audio>`); attach a
2305
+ ref for custom controls.
2306
+
2307
+ ```tsx
2308
+ import { AudioPlayer, type AudioPlayerHandle } from "@vincentt-xr/sdk";
2309
+
2310
+ const player = useRef<AudioPlayerHandle>(null);
2311
+
2312
+ <AudioPlayer
2313
+ ref={player}
2314
+ src={musicUrl}
2315
+ playMode="loop"
2316
+ loopCount={3}
2317
+ autoplay
2318
+ volume={80}
2319
+ onError={(error) => console.warn(error)}
2320
+ />;
2321
+
2322
+ // From a user gesture: player.current?.play(), .pause(), or .stop().
2323
+ ```