@vincentt-xr/sdk 2.0.0-alpha.1 → 2.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (352) hide show
  1. package/GROUNDING.md +1588 -20
  2. package/dist/{DebuggerUI-CatYc-RG.cjs → DebuggerUI-B45_Gb9d.cjs} +1 -1
  3. package/dist/{DebuggerUI-gLMKiVwJ.js → DebuggerUI-Cq28c18s.js} +1 -1
  4. package/dist/FaceMeshRenderer-6t8OsQhK.js +1195 -0
  5. package/dist/FaceMeshRenderer-lQLhYIW-.cjs +155 -0
  6. package/dist/FaceRetouch-CDrL0pXg.js +551 -0
  7. package/dist/FaceRetouch-DtNl8-4e.cjs +134 -0
  8. package/dist/audio/AudioEngine.d.ts +3 -1
  9. package/dist/audio/AudioPlayer.d.ts +29 -0
  10. package/dist/audio/AudioPlayerLogic.d.ts +4 -0
  11. package/dist/audio/index.d.ts +2 -0
  12. package/dist/components/Axes/index.d.ts +6 -0
  13. package/dist/components/FaceMesh/FaceMeshRenderer.d.ts +17 -0
  14. package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +17 -1
  15. package/dist/components/FaceMesh/canonical/FaceMeshMaterial.d.ts +25 -0
  16. package/dist/components/FaceMesh/canonical/assets.d.ts +3 -0
  17. package/dist/components/FaceMesh/canonical/geometryGroups.d.ts +9 -0
  18. package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +2 -0
  19. package/dist/components/FaceMesh/canonical/shaders.d.ts +2 -0
  20. package/dist/components/FaceMesh/deform.d.ts +3 -0
  21. package/dist/components/FaceMesh/index.d.ts +5 -0
  22. package/dist/components/FaceMesh/material/defaults.d.ts +70 -0
  23. package/dist/components/FaceMesh/material/index.d.ts +1 -0
  24. package/dist/components/FaceMesh/material/options.d.ts +5 -1
  25. package/dist/components/FaceMesh/material/support.d.ts +7 -2
  26. package/dist/components/FaceMesh/material/textures.d.ts +4 -0
  27. package/dist/components/FaceMesh/material/types.d.ts +2 -2
  28. package/dist/components/FaceRetouch/Background.d.ts +5 -6
  29. package/dist/components/FaceRetouch/FaceRetouch.d.ts +47 -0
  30. package/dist/components/FaceRetouch/index.d.ts +2 -8
  31. package/dist/components/FaceRetouch/landmarkMask.d.ts +0 -5
  32. package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +3 -7
  33. package/dist/components/FaceTracker/index.d.ts +5 -3
  34. package/dist/components/FaceWarp3D/FaceWarp3DHead.d.ts +1 -1
  35. package/dist/components/HandJoints2D/index.d.ts +1 -1
  36. package/dist/components/HandTracking2D/HandVisual.d.ts +1 -1
  37. package/dist/components/HandTracking2D/index.d.ts +1 -1
  38. package/dist/components/HandTracking2DController/index.d.ts +1 -1
  39. package/dist/components/HandTracking3D/index.d.ts +1 -1
  40. package/dist/components/LightObject/index.d.ts +20 -0
  41. package/dist/components/LookAt/index.d.ts +23 -0
  42. package/dist/components/MeshRenderer/index.d.ts +58 -0
  43. package/dist/components/PerspectiveCamera/index.d.ts +2 -8
  44. package/dist/components/PinToMesh/index.d.ts +47 -0
  45. package/dist/components/RenderGroupLayer/index.d.ts +6 -0
  46. package/dist/components/SceneObjectRenderer/index.d.ts +16 -0
  47. package/dist/components/Screen2D/ScreenTransformFrame.d.ts +11 -0
  48. package/dist/components/Screen2D/ScreenTransformHelpers.d.ts +19 -0
  49. package/dist/components/Screen2D/ScreenTransformInteraction.d.ts +18 -0
  50. package/dist/components/Screen2D/index.d.ts +5 -0
  51. package/dist/components/ScreenGlass/defaults.d.ts +3 -0
  52. package/dist/components/ScreenGlass/index.d.ts +5 -0
  53. package/dist/components/ScreenGlass/shaders.d.ts +2 -0
  54. package/dist/components/ScreenGlass/types.d.ts +10 -0
  55. package/dist/components/ScreenGradient/defaults.d.ts +5 -0
  56. package/dist/components/ScreenGradient/index.d.ts +5 -0
  57. package/dist/components/ScreenGradient/shaders.d.ts +2 -0
  58. package/dist/components/ScreenGradient/types.d.ts +19 -0
  59. package/dist/components/ScreenImage/ScreenImage.d.ts +4 -0
  60. package/dist/components/ScreenImage/components/DynamicTextureSync.d.ts +9 -0
  61. package/dist/components/ScreenImage/components/ScreenImageBody.d.ts +9 -0
  62. package/dist/components/ScreenImage/components/ScreenImageFrame.d.ts +9 -0
  63. package/dist/components/ScreenImage/components/ScreenImageTransformFrame.d.ts +9 -0
  64. package/dist/components/ScreenImage/fit.d.ts +52 -0
  65. package/dist/components/ScreenImage/index.d.ts +8 -87
  66. package/dist/components/ScreenImage/materials/ScreenImageBlendMaterial.d.ts +9 -0
  67. package/dist/components/ScreenImage/materials/ScreenImageMatcap.d.ts +3 -0
  68. package/dist/components/ScreenImage/source.d.ts +16 -0
  69. package/dist/components/ScreenImage/types.d.ts +37 -0
  70. package/dist/components/ScreenImage/utils/ScreenImageConstants.d.ts +9 -0
  71. package/dist/components/ScreenImage/utils/ScreenImageHelpers.d.ts +4 -0
  72. package/dist/components/ScreenShape/geometry.d.ts +5 -0
  73. package/dist/components/ScreenShape/index.d.ts +5 -0
  74. package/dist/components/ScreenShape/types.d.ts +41 -0
  75. package/dist/components/ScreenSpaceUI/ScreenTransformGuide.d.ts +15 -0
  76. package/dist/components/ScreenSpaceUI/ScreenTransformOverlayGuide.d.ts +9 -0
  77. package/dist/components/ScreenSpaceUI/index.d.ts +2 -0
  78. package/dist/components/ScreenText/components/ScreenTextBody.d.ts +2 -0
  79. package/dist/components/ScreenText/components/ScreenTextTransformFrame.d.ts +9 -0
  80. package/dist/components/ScreenText/constants.d.ts +5 -0
  81. package/dist/components/ScreenText/index.d.ts +6 -20
  82. package/dist/components/ScreenText/types.d.ts +31 -0
  83. package/dist/components/ScreenText/utils/ScreenTextHelpers.d.ts +14 -0
  84. package/dist/components/ScreenText/utils/ScreenTextMappings.d.ts +17 -0
  85. package/dist/components/SequentialRenderPasses/index.d.ts +6 -0
  86. package/dist/components/TextLabel/TextLabel.d.ts +5 -0
  87. package/dist/components/TextLabel/alignment.d.ts +4 -0
  88. package/dist/components/TextLabel/bend.d.ts +9 -0
  89. package/dist/components/TextLabel/canvas/buildTextLabelTexture.d.ts +4 -0
  90. package/dist/components/TextLabel/canvas/canvasTypography.d.ts +13 -0
  91. package/dist/components/TextLabel/canvas/drawBackground.d.ts +21 -0
  92. package/dist/components/TextLabel/canvas/drawEffects.d.ts +34 -0
  93. package/dist/components/TextLabel/canvas/drawText.d.ts +19 -0
  94. package/dist/components/TextLabel/components/TextLabelMesh.d.ts +13 -0
  95. package/dist/components/TextLabel/constants.d.ts +2 -0
  96. package/dist/components/TextLabel/hooks/useTextLabelTexture.d.ts +7 -0
  97. package/dist/components/TextLabel/index.d.ts +2 -161
  98. package/dist/components/TextLabel/layout/measurement.d.ts +8 -0
  99. package/dist/components/TextLabel/layout/resolveTextLayout.d.ts +25 -0
  100. package/dist/components/TextLabel/layout/wrapText.d.ts +6 -0
  101. package/dist/components/TextLabel/types.d.ts +227 -0
  102. package/dist/components/TrackingAnchor/index.d.ts +18 -2
  103. package/dist/components/Transform3D/index.d.ts +30 -0
  104. package/dist/components/XRMediaSurface/index.d.ts +6 -6
  105. package/dist/components/XRViewport/XRViewport.d.ts +11 -0
  106. package/dist/components/XRViewport/components/HiddenMedia.d.ts +7 -0
  107. package/dist/components/XRViewport/components/MediaAssetPicker.d.ts +8 -0
  108. package/dist/components/XRViewport/components/MediaControlStack.d.ts +13 -0
  109. package/dist/components/XRViewport/components/XRViewportLayers.d.ts +30 -0
  110. package/dist/components/XRViewport/constants.d.ts +24 -0
  111. package/dist/components/XRViewport/contexts.d.ts +6 -0
  112. package/dist/components/XRViewport/hooks/useViewportExternalFaces.d.ts +13 -0
  113. package/dist/components/XRViewport/hooks/useViewportMediaSource.d.ts +16 -0
  114. package/dist/components/XRViewport/hooks/useViewportMediaTexture.d.ts +10 -0
  115. package/dist/components/XRViewport/hooks/useViewportPipeline.d.ts +41 -0
  116. package/dist/components/XRViewport/hooks/useViewportRenderState.d.ts +47 -0
  117. package/dist/components/XRViewport/index.d.ts +3 -118
  118. package/dist/components/XRViewport/layout.d.ts +16 -0
  119. package/dist/components/XRViewport/slots.d.ts +16 -0
  120. package/dist/components/XRViewport/types.d.ts +89 -0
  121. package/dist/components/XRViewport/useViewportSize.d.ts +3 -0
  122. package/dist/components/index.d.ts +12 -0
  123. package/dist/contexts-BKvTIayr.cjs +1 -0
  124. package/dist/contexts-Bk5zz_VR.js +214 -0
  125. package/dist/debug-ui/component-editor/index.d.ts +19 -0
  126. package/dist/debug-ui/face-mesh-renderer-settings/index.d.ts +8 -0
  127. package/dist/debug-ui/face-retouch/index.d.ts +11 -0
  128. package/dist/debug-ui/hand-joints-2d/index.d.ts +1 -1
  129. package/dist/debug-ui/hand-tracking-2d/index.d.ts +1 -1
  130. package/dist/debug-ui/hand-tracking-2d-controller/index.d.ts +1 -1
  131. package/dist/debug-ui/hand-tracking-3d/index.d.ts +1 -1
  132. package/dist/debug-ui/hierarchy-panel/index.d.ts +29 -0
  133. package/dist/debug-ui/index.d.ts +25 -2
  134. package/dist/debug-ui/layer-panel/index.d.ts +3 -2
  135. package/dist/debug-ui/media-source/index.d.ts +2 -0
  136. package/dist/debug-ui/media-source/session.d.ts +12 -0
  137. package/dist/debug-ui/media-source.cjs +1 -1
  138. package/dist/debug-ui/media-source.js +6 -4
  139. package/dist/debug-ui/mesh-renderer-settings/index.d.ts +6 -0
  140. package/dist/debug-ui/object-inspector-panel/index.d.ts +25 -0
  141. package/dist/debug-ui/pin-to-mesh-settings/index.d.ts +10 -0
  142. package/dist/debug-ui/scene-editor/context.d.ts +38 -0
  143. package/dist/debug-ui/scene-editor/hierarchy-panel.d.ts +9 -0
  144. package/dist/debug-ui/scene-editor/index.d.ts +11 -0
  145. package/dist/debug-ui/scene-editor/inspector-panel.d.ts +7 -0
  146. package/dist/debug-ui/scene-editor/store-utils.d.ts +2 -0
  147. package/dist/debug-ui/scene-editor/viewport-toolbar.d.ts +4 -0
  148. package/dist/debug-ui/scene-editor/viewport.d.ts +21 -0
  149. package/dist/debug-ui/screen-glass-settings/index.d.ts +6 -0
  150. package/dist/debug-ui/screen-gradient-settings/GradientStopsEditor.d.ts +5 -0
  151. package/dist/debug-ui/screen-gradient-settings/index.d.ts +6 -0
  152. package/dist/debug-ui/screen-gradient-settings/styles.d.ts +3 -0
  153. package/dist/debug-ui/screen-image-settings/index.d.ts +4 -2
  154. package/dist/debug-ui/screen-shape-settings/index.d.ts +6 -0
  155. package/dist/debug-ui/screen-text-settings/NumberField.d.ts +9 -0
  156. package/dist/debug-ui/screen-text-settings/ScreenTextSettingsPanel.d.ts +3 -0
  157. package/dist/debug-ui/screen-text-settings/controls/NestedSection.d.ts +7 -0
  158. package/dist/debug-ui/screen-text-settings/controls/Section.d.ts +8 -0
  159. package/dist/debug-ui/screen-text-settings/controls/icons.d.ts +4 -0
  160. package/dist/debug-ui/screen-text-settings/index.d.ts +2 -122
  161. package/dist/debug-ui/screen-text-settings/sections/BackgroundSection.d.ts +8 -0
  162. package/dist/debug-ui/screen-text-settings/sections/ContentSection.d.ts +5 -0
  163. package/dist/debug-ui/screen-text-settings/sections/EffectsSection.d.ts +25 -0
  164. package/dist/debug-ui/screen-text-settings/sections/StyleSection.d.ts +5 -0
  165. package/dist/debug-ui/screen-text-settings/sections/TransformSection.d.ts +5 -0
  166. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundFillControls.d.ts +2 -0
  167. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundSizeControls.d.ts +2 -0
  168. package/dist/debug-ui/screen-text-settings/sections/background/BackgroundStrokeControls.d.ts +2 -0
  169. package/dist/debug-ui/screen-text-settings/sections/background/types.d.ts +7 -0
  170. package/dist/debug-ui/screen-text-settings/sections/effects/BendEffectSection.d.ts +4 -0
  171. package/dist/debug-ui/screen-text-settings/sections/effects/EffectChrome.d.ts +21 -0
  172. package/dist/debug-ui/screen-text-settings/sections/effects/GlowEffectSection.d.ts +4 -0
  173. package/dist/debug-ui/screen-text-settings/sections/effects/InnerShadowEffectSection.d.ts +4 -0
  174. package/dist/debug-ui/screen-text-settings/sections/effects/OutlineEffectSection.d.ts +4 -0
  175. package/dist/debug-ui/screen-text-settings/sections/effects/ShadowEffectSection.d.ts +4 -0
  176. package/dist/debug-ui/screen-text-settings/sections/effects/types.d.ts +14 -0
  177. package/dist/debug-ui/screen-text-settings/settings.d.ts +90 -0
  178. package/dist/debug-ui/screen-text-settings/styles.d.ts +26 -0
  179. package/dist/debug-ui/screen-text-settings/transform.d.ts +6 -0
  180. package/dist/debug-ui/screen-text-settings/types.d.ts +9 -0
  181. package/dist/debug-ui/shared/ScreenTransformSection.d.ts +6 -3
  182. package/dist/debug-ui/transform-3d-settings/index.d.ts +11 -0
  183. package/dist/debug-ui.cjs +3 -13
  184. package/dist/debug-ui.js +7896 -5098
  185. package/dist/drawNormalizedCrop-B-7zU-Xw.cjs +1 -0
  186. package/dist/drawNormalizedCrop-CRdIDyl1.js +44 -0
  187. package/dist/entries/all.d.ts +1 -0
  188. package/dist/entries/core.d.ts +53 -6
  189. package/dist/entries/face-effects.d.ts +10 -8
  190. package/dist/entries/internal.d.ts +1 -1
  191. package/dist/entries/low-level.d.ts +18 -4
  192. package/dist/entries/scene-object.d.ts +10 -0
  193. package/dist/entries/tracking.d.ts +25 -2
  194. package/dist/face-action/index.d.ts +34 -0
  195. package/dist/face-attribute-expression/index.d.ts +10 -0
  196. package/dist/face-effects.cjs +1 -342
  197. package/dist/face-effects.js +1391 -2767
  198. package/dist/face-info/blendshapes.d.ts +3 -0
  199. package/dist/face-info/derive.d.ts +8 -0
  200. package/dist/face-info/drawFaceCrop.d.ts +10 -0
  201. package/dist/face-info/getFaceInfo.d.ts +8 -0
  202. package/dist/face-info/index.d.ts +8 -0
  203. package/dist/face-info/landmarks.d.ts +21 -0
  204. package/dist/face-info/types.d.ts +52 -0
  205. package/dist/getFaceInfo-CyPAKh3h.cjs +1 -0
  206. package/dist/getFaceInfo-hUKMLFcr.js +118 -0
  207. package/dist/{handJoints2d.d.ts → hand/handJoints2d.d.ts} +1 -1
  208. package/dist/{handTracking2d.d.ts → hand/handTracking2d.d.ts} +1 -1
  209. package/dist/{handTracking2dController.d.ts → hand/handTracking2dController.d.ts} +1 -1
  210. package/dist/{handTracking3d.d.ts → hand/handTracking3d.d.ts} +1 -1
  211. package/dist/hooks/faceResults.d.ts +1 -1
  212. package/dist/hooks/useCalibrationFaceTracker.d.ts +1 -1
  213. package/dist/hooks/useFaceAction.d.ts +3 -0
  214. package/dist/hooks/useFaceAttributeExpression.d.ts +16 -0
  215. package/dist/hooks/useFaceDetection.d.ts +18 -0
  216. package/dist/hooks/useFaceInfo.d.ts +7 -0
  217. package/dist/hooks/useFaceInfos.d.ts +15 -0
  218. package/dist/hooks/useFaceResultReader.d.ts +10 -0
  219. package/dist/hooks/useFrameCapture.d.ts +18 -0
  220. package/dist/hooks/useMediaRecorder.d.ts +14 -2
  221. package/dist/hooks/useOwnedTexture.d.ts +6 -0
  222. package/dist/hooks/useTrackerContext.d.ts +2 -2
  223. package/dist/hooks/useXRSession/facade.d.ts +3 -0
  224. package/dist/hooks/useXRSession/registry.d.ts +9 -0
  225. package/dist/index-82aENaRq.cjs +1 -0
  226. package/dist/{index-BGNbrc4C.js → index-BgQAX8rs.js} +213 -125
  227. package/dist/index-CEc1CAmB.cjs +1 -0
  228. package/dist/index-CTHeBQOs.js +178 -0
  229. package/dist/index-D4bLzBr6.cjs +1 -0
  230. package/dist/index-DSpzjuqd.cjs +1 -0
  231. package/dist/index-Di70KWuY.js +2049 -0
  232. package/dist/index-DzEzkBIH.cjs +17 -0
  233. package/dist/{index-safCs_af.js → index-JygePOwo.js} +565 -508
  234. package/dist/index-UC0Ux38E.js +111 -0
  235. package/dist/{internal-Cae1ps15.js → internal-DuMPpcbp.js} +86 -80
  236. package/dist/internal-i2JkM85N.cjs +1 -0
  237. package/dist/internal.cjs +1 -1
  238. package/dist/internal.js +2 -2
  239. package/dist/low-level.cjs +1 -1
  240. package/dist/low-level.js +1229 -86
  241. package/dist/main.cjs +1 -73
  242. package/dist/main.js +545 -1693
  243. package/dist/material/MaterialFill.d.ts +20 -0
  244. package/dist/material/MeshMaterial.d.ts +14 -0
  245. package/dist/material/index.d.ts +2 -0
  246. package/dist/models/Body/index.d.ts +1 -1
  247. package/dist/models/Face/index.d.ts +1 -0
  248. package/dist/models/Face/rotation.d.ts +3 -0
  249. package/dist/render/renderGroups.d.ts +47 -0
  250. package/dist/render/renderPasses.d.ts +18 -0
  251. package/dist/renderGroups-C9QJrAXu.js +152 -0
  252. package/dist/renderGroups-Dvm1J4lW.cjs +1 -0
  253. package/dist/renderLayers-lDtL0_43.cjs +1 -0
  254. package/dist/renderLayers-zrK2pv5O.js +13 -0
  255. package/dist/renderPasses-CT5iBwtg.cjs +21 -0
  256. package/dist/renderPasses-CjgG_6kQ.js +1500 -0
  257. package/dist/scene-object/index.d.ts +146 -0
  258. package/dist/scene-object/runtimeRegistry.d.ts +14 -0
  259. package/dist/scene-object.cjs +1 -0
  260. package/dist/scene-object.js +40 -0
  261. package/dist/{screenImageSettings.d.ts → screen-image/screenImageSettings.d.ts} +9 -2
  262. package/dist/{screenTextSettings.d.ts → screen-text/screenTextSettings.d.ts} +18 -2
  263. package/dist/screen2d/coordinates.d.ts +87 -0
  264. package/dist/screen2d/index.d.ts +3 -0
  265. package/dist/screen2d/screenTransform.d.ts +47 -0
  266. package/dist/screen2d/smoothing.d.ts +5 -0
  267. package/dist/{selectors-BLl2JUWW.cjs → selectors-2QvO5RYY.cjs} +1 -1
  268. package/dist/{selectors-CXd_x-6t.js → selectors-Dw3C-gSk.js} +3 -3
  269. package/dist/services/MetricsCollector/index.d.ts +7 -1
  270. package/dist/services/MetricsCollector/utils.d.ts +1 -1
  271. package/dist/session-BChR4TgK.cjs +31 -0
  272. package/dist/session-C1StwN0f.js +363 -0
  273. package/dist/texture/animatedTexture.d.ts +61 -0
  274. package/dist/texture/animationMath.d.ts +8 -0
  275. package/dist/texture/sourceResolver.d.ts +30 -0
  276. package/dist/tracking.cjs +1 -1
  277. package/dist/tracking.js +580 -321
  278. package/dist/types/face.d.ts +1 -13
  279. package/dist/types/index.d.ts +0 -1
  280. package/dist/types/nodes.d.ts +2 -1
  281. package/dist/types/session.d.ts +2 -0
  282. package/dist/types/tracking.d.ts +8 -2
  283. package/dist/types/util-types.d.ts +1 -1
  284. package/dist/useFaceInfo-C-WfMlYp.cjs +1 -0
  285. package/dist/useFaceInfo-CwsHZedC.js +62 -0
  286. package/dist/useFaceResults-CkFc1egZ.js +19 -0
  287. package/dist/useFaceResults-VcFdOUFV.cjs +1 -0
  288. package/dist/utils/drawNormalizedCrop.d.ts +22 -0
  289. package/dist/utils/integrity.d.ts +4 -25
  290. package/dist/utils/landmarkBounds.d.ts +1 -1
  291. package/dist/utils/logger.d.ts +2 -2
  292. package/docs/MIGRATION_GUIDE.md +589 -0
  293. package/docs/SDK_DOCUMENTATION.md +1155 -0
  294. package/docs/api/audio.mdx +36 -0
  295. package/docs/api/behavior-components.mdx +164 -0
  296. package/docs/api/capture-recording.mdx +103 -0
  297. package/docs/api/debug-ui-scene-editor.mdx +249 -0
  298. package/docs/api/face-actions.mdx +104 -0
  299. package/docs/api/face-attribute-expression.mdx +87 -0
  300. package/docs/api/face-binding-anchors.mdx +53 -0
  301. package/docs/api/face-effects.mdx +19 -0
  302. package/docs/api/face-info.mdx +112 -0
  303. package/docs/api/face-mesh.mdx +98 -0
  304. package/docs/api/face-painting.mdx +33 -0
  305. package/docs/api/face-retouching.mdx +73 -0
  306. package/docs/api/material-render-state.mdx +132 -0
  307. package/docs/api/runtime-3d.mdx +311 -0
  308. package/docs/api/scene-object-store.mdx +213 -0
  309. package/docs/api/screen-image.mdx +41 -0
  310. package/docs/api/screen-shape.mdx +64 -0
  311. package/docs/api/screen-space-coordinates.mdx +41 -0
  312. package/docs/api/screen-text.mdx +39 -0
  313. package/docs/api/texture-sources.mdx +55 -0
  314. package/docs/api/trackers.mdx +46 -0
  315. package/docs/docs.json +70 -0
  316. package/docs/examples/scene-object-recipes.mdx +194 -0
  317. package/docs/guides/harness-debug-ui.mdx +139 -0
  318. package/docs/inspector/hierarchy-panel.mdx +51 -0
  319. package/docs/inspector/object-inspector-panel.mdx +51 -0
  320. package/docs/introduction.mdx +121 -0
  321. package/docs/scene-objects/overview.mdx +127 -0
  322. package/docs/scene-objects/render-groups.mdx +77 -0
  323. package/docs/scene-objects/runtime-rendering.mdx +75 -0
  324. package/docs/texture-api-consumer.md +58 -0
  325. package/package.json +37 -7
  326. package/dist/MediaSourceSwitcher-C-xYg0nQ.js +0 -317
  327. package/dist/MediaSourceSwitcher-Cq3g4IKw.cjs +0 -31
  328. package/dist/components/FaceRetouch/defaults.d.ts +0 -13
  329. package/dist/components/FaceRetouch/retouchRegions.d.ts +0 -23
  330. package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +0 -12
  331. package/dist/defaults-BLpcN3pG.cjs +0 -1
  332. package/dist/defaults-LtY4M7dg.js +0 -55
  333. package/dist/index-BXyObVHF.cjs +0 -1
  334. package/dist/index-CY3bUAcW.js +0 -12
  335. package/dist/index-C_Duh13c.cjs +0 -1
  336. package/dist/index-D8fpPQkC.cjs +0 -1
  337. package/dist/index-DxshgBu0.cjs +0 -1
  338. package/dist/index-O-2crZ3M.js +0 -1102
  339. package/dist/internal-BjyhBP8i.cjs +0 -1
  340. package/dist/modules/XRClient/index.d.ts +0 -9
  341. package/dist/modules/index.d.ts +0 -1
  342. package/dist/screenImageSettings-6NRg4cx9.js +0 -39
  343. package/dist/screenImageSettings-y_B4_767.cjs +0 -1
  344. package/dist/services/ApiService.d.ts +0 -12
  345. package/dist/types/client.d.ts +0 -3
  346. package/dist/useFaceResults-cpsSmKKc.js +0 -81
  347. package/dist/useFaceResults-hVUJYSEY.cjs +0 -1
  348. package/dist/useXRModel-3DcUa28n.cjs +0 -1
  349. package/dist/useXRModel-BMJE1hrk.js +0 -28
  350. /package/dist/{layerStack.d.ts → render/layerStack.d.ts} +0 -0
  351. /package/dist/{screenImagePresets.d.ts → screen-image/screenImagePresets.d.ts} +0 -0
  352. /package/dist/{screenTextPresets.d.ts → screen-text/screenTextPresets.d.ts} +0 -0
package/GROUNDING.md CHANGED
@@ -10,7 +10,8 @@ or spelunk `node_modules` for the API.
10
10
  The API is split into task-domain entry points. Import each symbol from the door
11
11
  this reference shows for it; the imports here are literal.
12
12
 
13
- - `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useMediaRecorder`, `dataURLtoFile`), audio.
13
+ - `@vincentt-xr/sdk` — **core**: providers, scene, screen-space layout, runtime renderers (`Transform3D`, `MeshRenderer`, `SceneObjectRenderer`), `TextLabel`, `Panel`, `SpriteAnimation`, `VideoBackground`, `AspectRatioContainer`, capture/share (`useFrameCapture`, `useMediaRecorder`, `dataURLtoFile`), audio.
14
+ - `@vincentt-xr/sdk/scene-object` — **public authoring state**: scene-object/component types, factories, selectors, and immutable store operations.
14
15
  - `@vincentt-xr/sdk/tracking` — **trackers**: face/hand/gesture/body/segmentation, `TrackingAnchor`, `GestureTrigger`, the bare `FaceMesh`, `useFaceResults`.
15
16
  - `@vincentt-xr/sdk/face-effects` — **face deep**: FaceMesh material config, retouch, canonical mesh, head-binding.
16
17
  - `@vincentt-xr/sdk/low-level` — **escape hatch**: raw model-node reads, selector hooks, custom-tracker plumbing. Reach here only when core + tracking can't express it.
@@ -24,6 +25,50 @@ logic is R3F `useFrame`; per-mount setup is `useEffect`.
24
25
  > patterns) into the project's `GROUNDING.md`. Edit the SDK API here, in the repo
25
26
  > that owns the API.
26
27
 
28
+ ### Internal source layout
29
+
30
+ The public import doors above are stable. Internal files are organized by domain
31
+ so shared behavior stays reusable instead of duplicated in large components:
32
+
33
+ - `lib/components/ScreenImage/` — thin `ScreenImage.tsx` orchestrator plus
34
+ reusable body, frame, transform-frame, material, matcap, shader, source, and
35
+ helper modules.
36
+ - `lib/components/ScreenText/` — thin `ScreenText.tsx` orchestrator plus
37
+ reusable body, transform-frame, constants, mapping, and helper modules.
38
+ - `lib/components/TextLabel/` — low-level canvas text renderer split into
39
+ texture building, layout, style, hooks, and body modules.
40
+ - `lib/components/XRViewport/` — thin viewport orchestrator plus reusable
41
+ provider, stage, overlays, media, control, sizing, slot, and layout modules.
42
+ - `lib/components/Screen2D/` — shared screen-transform interaction/frame helpers
43
+ used by screen-space authoring components.
44
+ - `lib/debug-ui/screen-text-settings/` — panel orchestrator plus reusable
45
+ sections, controls, settings, styles, transform helpers, and types.
46
+ - `lib/render/` — render-group, render-pass, and layer-stack internals.
47
+ - `lib/hand/`, `lib/screen-image/`, and `lib/screen-text/` — domain-level
48
+ primitives, presets, settings, and tests that used to live flat in `lib/`.
49
+
50
+ When adding features, keep public exports at the documented import doors and put
51
+ implementation details in the matching domain folder. Do not reintroduce
52
+ component-local duplicate shader, transform, text-rendering, or settings logic.
53
+
54
+ ### Example asset layout
55
+
56
+ Example-app assets are organized by purpose, not by whichever page first needed
57
+ them:
58
+
59
+ - `example/src/assets/media/images/` — reusable still textures and sample images.
60
+ - `example/src/assets/media/images/people/` — face/body tracking sample stills.
61
+ - `example/src/assets/media/images/screenshots/` — screenshot reference images.
62
+ - `example/src/assets/media/videos/` — sample MP4/WebM sources.
63
+ - `example/src/assets/media/gifs/` — animated GIF source files.
64
+ - `example/src/assets/media/sequences/<demo>/` — ordered image-sequence frames.
65
+ - `example/src/assets/models/` — GLB files used by example 3D scenes.
66
+ - `example/src/assets/face-effects/` — domain-owned face-effect assets.
67
+
68
+ Do not flatten all example assets into the root. Do not add new `utils/` media
69
+ folders. Compression is a separate visual/behavior review because it can change
70
+ tracking fixtures and texture output.
71
+
27
72
  ---
28
73
 
29
74
  ## Screen-space layout (the primary tool)
@@ -94,6 +139,89 @@ authoring prefer `position`, `size`, and `rotation`. `anchors` still exist as
94
139
  the lower-level layout escape hatch. If no source is provided, it renders the
95
140
  SDK-owned square default image.
96
141
 
142
+ ### `<ScreenShape>` — editable vector primitives
143
+
144
+ Use `ScreenShape` for circles, ellipses, rectangles, rounded rectangles, regular
145
+ polygons, stars, and lines. Do not generate raster image assets for these basic
146
+ shapes. Placement and drag editing use the shared `ScreenTransform` settings.
147
+ Circles, regular polygons, and stars preserve equal axes inside non-square
148
+ transforms; ellipses and rectangles fill the transform.
149
+
150
+ ```tsx
151
+ import { ScreenShape, ScreenSpaceUI } from "@vincentt-xr/sdk";
152
+
153
+ <ScreenSpaceUI>
154
+ <ScreenShape
155
+ transform={{ position: { x: 0, y: 0 }, size: { width: 220, height: 220 } }}
156
+ settings={{
157
+ kind: "star",
158
+ fillColor: "#ffd54a",
159
+ strokeColor: "#ffffff",
160
+ strokeWidth: 4,
161
+ sides: 5,
162
+ innerRadiusRatio: 0.45,
163
+ }}
164
+ />
165
+ </ScreenSpaceUI>;
166
+ ```
167
+
168
+ Scene objects pair `ScreenTransform` with `ScreenShape`. Import scene-object
169
+ factories from `@vincentt-xr/sdk/scene-object` and the live
170
+ `ScreenShapeSettingsPanel` from `@vincentt-xr/sdk/debug-ui`.
171
+
172
+ Settings: `kind`; `fillMode`, `fillColor`, `fillOpacity`; `strokeColor`,
173
+ `strokeOpacity`, `strokeWidth`, `dashed`; `cornerRadius` for rounded rectangles;
174
+ `sides` for polygons/stars; `innerRadiusRatio` for stars; and normalized
175
+ `lineStart`/`lineEnd` `{ x, y }` coordinates for lines.
176
+
177
+ Set `fillMode: "gradient"` and provide `gradient: { type, stops, angle, center,
178
+ radius }`. Types are `linear`, `radial`, and `angular` (conic). A gradient accepts
179
+ 2–8 `{ offset, color }` stops. `ScreenGradientMaterial` is a separate reusable
180
+ component exported from core; its settings panel is exported from debug-ui.
181
+
182
+ Set `fillMode: "glass"` and provide `glass: { tint, opacity, frost, highlight }`
183
+ for lightweight translucent glass. `ScreenGlassMaterial` is a separate reusable
184
+ component. It is a stylized overlay material, not backdrop blur/refraction.
185
+
186
+ ### Shared material primitives
187
+
188
+ Import shared material primitives from `@vincentt-xr/sdk` only:
189
+
190
+ ```tsx
191
+ import {
192
+ DEFAULT_MATERIAL_FILL,
193
+ MaterialFill,
194
+ MeshMaterial,
195
+ createTwoStopGradient,
196
+ type MaterialFillSettings,
197
+ type MeshMaterialSettings,
198
+ } from "@vincentt-xr/sdk";
199
+ ```
200
+
201
+ Use `MaterialFill` when a component needs the same solid/gradient/texture fill
202
+ system as `ScreenShape` and FaceMesh masks. `MaterialFillSettings.source` is
203
+ `"solid" | "gradient" | "texture"`; pass `color`, `opacity`, optional
204
+ `textureUrl`, and optional `gradient`.
205
+
206
+ ```tsx
207
+ const fill: MaterialFillSettings = {
208
+ ...DEFAULT_MATERIAL_FILL,
209
+ source: "gradient",
210
+ color: "#38bdf8",
211
+ opacity: 0.8,
212
+ gradient: createTwoStopGradient("#38bdf8"),
213
+ };
214
+
215
+ <mesh>
216
+ <planeGeometry args={[1, 1]} />
217
+ <MaterialFill settings={fill} />
218
+ </mesh>;
219
+ ```
220
+
221
+ Use `MeshMaterial` for serialized mesh material settings shared by
222
+ `MeshRenderer`; do not re-create local `<meshBasicMaterial>` /
223
+ `<meshStandardMaterial>` branches in scene-object components.
224
+
97
225
  ```tsx
98
226
  import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
99
227
 
@@ -147,9 +275,9 @@ Settings:
147
275
  position, size, rotation, pivot, render order, overlay/depth behavior, and visibility.
148
276
  - `DEFAULT_SCREEN_IMAGE_SETTINGS.image`: source kind, src, fit, opacity, color,
149
277
  flips, transparency, and alpha test.
150
- - `sourceKind` includes future media categories (`video`, `sequence`, `gif`),
151
- but the current component implementation supports `default`, static `image`,
152
- and already-loaded Three `texture`.
278
+ - The component source union is intentionally small: `default`, static `image`,
279
+ and an already-loaded Three `texture`. Video/GIF/sequence providers resolve
280
+ their media to a live `THREE.Texture` and pass it as `kind: "texture"`.
153
281
 
154
282
  Use `<ScreenImage>` instead of hand-authoring `<ScreenTransform>` + `<mesh>`
155
283
  when the object is conceptually a 2D screen image. Use the lower-level mesh path
@@ -168,6 +296,31 @@ including `name`, `kind: "screenImage"`, `transform`, `image`, `material`, and
168
296
  `screenSpaceUI`. The `transform` includes `coordinateSpace` and `canvasSize`.
169
297
  Use that payload as the handoff from human layout tuning back to code.
170
298
 
299
+ ### ScreenImage contract for every caller
300
+
301
+ `ScreenImage` is the canonical screen-image object in SDK and client apps. Any
302
+ debug panel, hierarchy row, editor, or runtime controller that targets one must
303
+ operate on the same mounted object state:
304
+
305
+ - Keep one source of truth for `transform`, source/material settings, and
306
+ visibility. Panel edits must immediately update the rendered object.
307
+ - Always wire `onScreenTransformChange` for inspector or editor usage. It is the
308
+ real drag/place path; do not implement a panel-only shadow transform.
309
+ - Use `coordinateSpace="canvas"` plus `canvasSize` for authoring tools, then
310
+ map canvas values to `position`/`size`; never treat design pixels as world
311
+ units. `fit`/`stretchMode` changes content fitting, not the object's bounds.
312
+ - Source replacement must preserve the object identity when possible, stop and
313
+ dispose the previous owned provider, and keep the current transform/material
314
+ settings. Dynamic video/canvas textures must continue marking fitted clones
315
+ for GPU upload.
316
+ - Respect capability-gated controls: static images expose image/material
317
+ controls only; animated providers expose only controls they support. Loading,
318
+ error, retry, pause, ended, and cleanup state must be visible to diagnostics.
319
+ - `visible`, `overlay`, `renderOrder`, `transparent`, `alphaTest`, blend mode,
320
+ opacity, tint, flips, pivot, rotation, size, and position are all part of the
321
+ live object contract and must remain synchronized from SDK or client entry
322
+ points.
323
+
171
324
  ### `<SpriteAnimation sheet={{ url, cols, rows, frames, fps }}>` — Animated transparent overlay
172
325
 
173
326
  The primitive for an animated transparent overlay (a frame border, a gesture-instruction graphic, an animated sticker). One transparent WebP atlas of `cols`×`rows` cells; the first `frames` play left-to-right, top-to-bottom at `fps`. It is a single static GPU texture with a per-frame UV offset — no video decode — so it works in every native and in-app browser and costs ~zero heat.
@@ -232,9 +385,190 @@ For local tuning, import `FaceWarp3DSettingsPanel` from
232
385
  `@vincentt-xr/sdk/debug-ui`. It is a draggable debug panel and should not
233
386
  be included in the final production scene.
234
387
 
388
+ ### `FaceMeshRenderer` — scene-object live 3D face mesh
389
+
390
+ Use `FaceMeshRenderer` for the standard SDK face-mesh scene-object component.
391
+ It owns face tracking internally through `FaceTracker`, reads live landmarks,
392
+ deforms the canonical face mesh, and renders through normal face-mesh material
393
+ settings. Prefer this over direct `CanonicalFaceMeshRenderer` in new
394
+ scene-object/editor integrations.
395
+
396
+ Import runtime/material symbols from `@vincentt-xr/sdk/face-effects`:
397
+
398
+ ```tsx
399
+ import {
400
+ DEFAULT_FACE_MESH_RENDERER,
401
+ DEFAULT_FACE_MESH_TRANSFORM,
402
+ FACE_MESH_MATERIAL_TYPES,
403
+ FACE_MESH_SIDES,
404
+ FaceMeshRenderer,
405
+ mapFaceMeshRendererSettingsToProps,
406
+ resolveFaceMeshRendererSettings,
407
+ type FaceMeshRendererSettings,
408
+ } from "@vincentt-xr/sdk/face-effects";
409
+ ```
410
+
411
+ Import scene-object defaults and store helpers from
412
+ `@vincentt-xr/sdk/scene-object`:
413
+
414
+ ```tsx
415
+ import {
416
+ createDefaultComponent,
417
+ createDefaultFaceMeshRendererSettings,
418
+ DEFAULT_RENDER_GROUP_IDS,
419
+ type RenderGroupSceneObjectStore,
420
+ } from "@vincentt-xr/sdk/scene-object";
421
+ ```
422
+
423
+ `FaceMeshRendererSettings` is the serialized component settings contract:
424
+ `enabled`, `faceIndex`, `targetFps`, `showLeftEyeGeometry`,
425
+ `showRightEyeGeometry`, `showMouthGeometry`, `showNoseGeometry`,
426
+ `showOverlayMask`, `showMouthMask`, `showLeftEyeMask`, `showRightEyeMask`,
427
+ `showNoseMask`, `externalMeshUrl`, `externalScale`, `materialType`,
428
+ `shaderPreset`, `textureMap`, `alphaMap`, `alphaOpacity`, `side`, `blendMode`,
429
+ `blendOpacity`, `baseColor`, `opacity`, `compositeShading`,
430
+ `maskTextureSource`, `maskSourceUrl`, `maskColor`, `maskOpacity`,
431
+ `faceTextureUrl`, `faceTextureStyle2Url`, `faceAlphaMaskUrl`, and
432
+ `renderOrder`.
433
+
434
+ `maskTextureSource` is `"solid" | "gradient" | "imageUrl" | "mediaUrl"`.
435
+ Solid and gradient masks use `maskColor`; image/media masks use
436
+ `maskSourceUrl`. `maskOpacity` is always `0..1`.
437
+
438
+ `materialType: "shader"` has three shader presets: `"basicShader"`,
439
+ `"originalFace"`, and `"composite"`. Use
440
+ `resolveFaceMeshRendererSettings(current, patch)` when editing settings so
441
+ shader/material/texture state stays synchronized. Selecting composite applies
442
+ the SDK composite defaults (`textureMap: "canonical"`, hard-light blend,
443
+ `blendOpacity: 0.85`, `compositeShading: true`) instead of silently falling
444
+ back to original-face rendering.
445
+
446
+ `externalMeshUrl` is for canonical-compatible OBJ assets that match the
447
+ renderer's live landmark deformation path. Third-party Face Mesh OBJ files are
448
+ reference assets only until the SDK has an explicit topology mapper from those
449
+ vertices to the 468 tracked face landmarks. Do not bundle OBJ/FBX/PNG reference
450
+ files in `lib/` or examples unless runtime code consumes them correctly.
451
+
452
+ Do not add settings that are not runtime-effective. Skull/ear geometry,
453
+ expression multiplier, and UV1 mapping require parser/runtime support before
454
+ they become public SDK settings.
455
+
456
+ Scene-object example:
457
+
458
+ ```tsx
459
+ const objectId = "face-mesh";
460
+ const transform = createDefaultComponent("Transform", objectId);
461
+ transform.settings.position.y = DEFAULT_FACE_MESH_TRANSFORM.position.y;
462
+
463
+ const faceMesh = createDefaultComponent("FaceMeshRenderer", objectId);
464
+ faceMesh.settings = {
465
+ ...createDefaultFaceMeshRendererSettings(),
466
+ materialType: FACE_MESH_MATERIAL_TYPES.wireframe,
467
+ side: FACE_MESH_SIDES.double,
468
+ baseColor: "#00e5ff",
469
+ opacity: 0.82,
470
+ } satisfies FaceMeshRendererSettings;
471
+
472
+ const store: RenderGroupSceneObjectStore = {
473
+ objects: {
474
+ [objectId]: {
475
+ id: objectId,
476
+ name: "Face Mesh",
477
+ enabled: true,
478
+ renderGroupId: DEFAULT_RENDER_GROUP_IDS.scene3d,
479
+ componentIds: [transform.id, faceMesh.id],
480
+ },
481
+ },
482
+ components: {
483
+ [transform.id]: transform,
484
+ [faceMesh.id]: faceMesh,
485
+ },
486
+ renderGroups: {
487
+ [DEFAULT_RENDER_GROUP_IDS.scene3d]: {
488
+ id: DEFAULT_RENDER_GROUP_IDS.scene3d,
489
+ name: "3D Scene",
490
+ domain: "scene3d",
491
+ enabled: true,
492
+ objectIds: [objectId],
493
+ },
494
+ },
495
+ renderGroupOrder: [DEFAULT_RENDER_GROUP_IDS.scene3d],
496
+ };
497
+ ```
498
+
499
+ When `SceneObjectRenderer` sees both `Transform` and `FaceMeshRenderer` on the
500
+ same object, it passes the authored `Transform` into `FaceMeshRenderer`.
501
+ FaceMesh does not use normal world-space `Transform3D` semantics. It applies
502
+ the supported Transform values in MediaPipe landmark space before camera
503
+ projection; it must never wrap the already-projected mesh in a parent
504
+ `Transform3D`, because that causes depth-dependent screen skew.
505
+
506
+ Supported FaceMesh Transform values are position and uniform scale only.
507
+ Rotation and non-uniform scale are ignored by the runtime until the SDK has a
508
+ real object-space face solve. Authored `position.y` is Y-up, but MediaPipe image
509
+ space is Y-down, so the renderer subtracts Y while adding X/Z.
510
+
511
+ This is the SDK's strict FaceMesh contract: FaceInfo/FaceBinding pose first,
512
+ FaceMesh deformation in face landmark space, supported authored Transform in
513
+ that space, then world/camera projection.
514
+ `DEFAULT_FACE_MESH_TRANSFORM` starts from `DEFAULT_TRANSFORM_3D` and sets
515
+ `position.y` to `0.001` as an intentional FaceMesh authoring bias, not as a
516
+ projection correction. Author a Transform with `position.y = 0` to opt out.
517
+
518
+ `mapFaceMeshRendererSettingsToProps(settings)` maps serialized
519
+ `FaceMeshRendererSettings` to low-level `CanonicalFaceMeshRendererProps`.
520
+ Use it only when bridging custom runtime code; ordinary scene-object rendering
521
+ should use `SceneObjectRenderer`.
522
+
523
+ `CanonicalFaceMeshRenderer` remains exported from `@vincentt-xr/sdk/face-effects`
524
+ for backwards compatibility and low-level experiments, but new editor and
525
+ scene-object work should use `FaceMeshRenderer`.
526
+
527
+ ### `FaceRetouch` — legacy landmark skin retouch
528
+
529
+ Use `FaceRetouch` from `@vincentt-xr/sdk/face-effects` for the legacy
530
+ landmark-masked skin retouch shader. The public controls are intentionally
531
+ minimal:
532
+
533
+ ```tsx
534
+ import {
535
+ FaceRetouch,
536
+ patchFaceRetouchControls,
537
+ resolveFaceRetouchControls,
538
+ type FaceRetouchControls,
539
+ } from "@vincentt-xr/sdk/face-effects";
540
+ import { FaceRetouchSettingsPanel } from "@vincentt-xr/sdk/debug-ui";
541
+
542
+ const [controls, setControls] = useState(() => resolveFaceRetouchControls());
543
+
544
+ const updateControls = (next: Partial<FaceRetouchControls>) => {
545
+ setControls((current) => patchFaceRetouchControls(current, next));
546
+ };
547
+
548
+ <FaceRetouch {...controls} />;
549
+ <FaceRetouchSettingsPanel value={controls} onChange={updateControls} />;
550
+ ```
551
+
552
+ `FaceRetouchControls` has only `skinSmoothing`, `skinRealism`,
553
+ `blemishReduction`, and `previewMask`. Do not invent provider, quality,
554
+ redness, teeth, eye, or blur controls for this component.
555
+
556
+ For multiple people, render one `FaceRetouch` per face:
557
+
558
+ ```tsx
559
+ <FaceRetouch {...controls} faceIndex={0} numFaces={2} />
560
+ <FaceRetouch {...controls} faceIndex={1} numFaces={2} />
561
+ ```
562
+
563
+ `numFaces` defaults to at least `faceIndex + 1`. `clipMinX` and `clipMaxX` are
564
+ optional horizontal UV clipping props in `0..1` space, useful for split-screen
565
+ comparison demos.
566
+
235
567
  ### `<FaceTracker>` — Face tracking (468 landmarks)
236
568
 
237
- Detects faces and provides face transform to children via context. Supports `faceIndex` (default 0) and `scaleWithFace`.
569
+ Detects faces and provides face transform to children via context. Supports
570
+ `faceIndex` (default `0`), `scaleWithFace`, and `followFaceRoll` (default
571
+ `true`; set `false` to keep children upright while the head tilts).
238
572
 
239
573
  ```tsx
240
574
  import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
@@ -249,6 +583,274 @@ import { FaceTracker, TrackingAnchor } from "@vincentt-xr/sdk/tracking";
249
583
  </FaceTracker>;
250
584
  ```
251
585
 
586
+ ### `FaceInfo` — face-driven effect logic
587
+
588
+ Use `FaceInfo` for head-pose, expression, landmark, bounds, and face-scale logic.
589
+ Do not read raw MediaPipe results or repeat landmark indices unless the facade
590
+ does not expose data required by a research/debugging tool.
591
+
592
+ ```tsx
593
+ import { hasFaceExpression, useFaceInfo } from "@vincentt-xr/sdk/tracking";
594
+
595
+ function FaceLogic() {
596
+ const face = useFaceInfo({ active: true });
597
+ if (!face) return null;
598
+
599
+ const mouthOpen = hasFaceExpression(face, "jawOpen");
600
+ const lookingLeft = face.pose.rotation.y > 0.2;
601
+ return <group visible={mouthOpen} rotation-y={lookingLeft ? 0.2 : 0} />;
602
+ }
603
+ ```
604
+
605
+ | Field | Coordinate contract |
606
+ | --------------- | ---------------------------------------------------------------- |
607
+ | `index` | Current result-array index; not a persistent identity |
608
+ | `landmarks` | MediaPipe normalized tracker-input coordinates, top-left origin |
609
+ | `bounds` | Normalized tracker-input bounds, top-left origin |
610
+ | `pose.position` | Existing face-tracker Three.js world position |
611
+ | `pose.rotation` | Radians: X pitch (+up), Y yaw (+left), Z roll |
612
+ | `pose.scale` | Smoothed uniform sizing heuristic used by face anchors/occluders |
613
+ | `eyeDistance` | Outer-eye distance in normalized tracker-input coordinates |
614
+ | `blendshapes` | MediaPipe category names and scores in `[0, 1]` |
615
+
616
+ `useFaceInfo({ faceIndex })` is the normal single-face React reader.
617
+ `useFaceInfos({ numFaces })` publishes all current faces. Both acquire the shared
618
+ face model through the SDK registry and use the existing throttle/empty-frame
619
+ hold lifecycle; do not mount a separate result polling loop. Call them inside
620
+ `XRProvider` and the R3F Canvas. Options are `active`, `numFaces`, `targetFps`,
621
+ `throttleMs`, and `holdMs`; `useFaceInfo` additionally accepts `faceIndex`.
622
+
623
+ ```tsx
624
+ function MultiFaceLogic() {
625
+ const faces = useFaceInfos({ numFaces: 3 });
626
+ return <group visible={faces.some((face) => hasFaceExpression(face, "jawOpen"))} />;
627
+ }
628
+ ```
629
+
630
+ For `useFrame` code that already has a low-level face node, call
631
+ `getFaceInfo(node, faceIndex, target)`. Create `target` once with
632
+ `createFaceInfoTarget()` and reuse it to avoid per-frame allocations. Use
633
+ `getFaceInfo` and `createFaceInfoTarget` from `@vincentt-xr/sdk/low-level`; they
634
+ are node-in escape hatches, not the normal React API. Use `getFaceBlendshape`
635
+ and `hasFaceExpression`; absent scores resolve to `0`, and the default expression
636
+ threshold is `DEFAULT_FACE_EXPRESSION_THRESHOLD`. Bind the R3F camera once with
637
+ `node.setCamera(camera, faceIndex)` before reading pose; the React hooks do this
638
+ automatically.
639
+
640
+ All supported expression names are typed by `FaceBlendshapeName` and listed in
641
+ the greppable `FACE_BLENDSHAPE_NAMES` constant exported from
642
+ `@vincentt-xr/sdk/tracking`. This includes `eyeBlinkLeft`, `eyeBlinkRight`,
643
+ `eyeWideLeft`, `eyeWideRight`, and `jawOpen`; do not guess string names.
644
+
645
+ Use `normalizedToScreenPixels` with the tracker input/display viewport size to
646
+ display a face landmark in Screen Image/Text/Shape coordinates. The tracker
647
+ input is already the SDK feed; do not apply media cover/mirroring a second time.
648
+ Never mutate or mirror `FaceInfo.landmarks` in place, manually project them,
649
+ infer a persistent face ID, or copy numeric landmark indices/thresholds into
650
+ application code.
651
+
652
+ ```ts
653
+ const nose = normalizedToScreenPixels({
654
+ point: face.landmarks[FACE_LANDMARK_INDICES["face.noseTip"]],
655
+ viewportSize,
656
+ });
657
+ ```
658
+
659
+ Effect House mapping: `FaceInfo.landmarks` replaces `pointsArray`, `bounds`
660
+ replaces `rect`, `pose.rotation` provides pitch/yaw/roll, and `eyeDistance` is
661
+ named the same. Use Face Actions below for the semantic `FaceAction` role.
662
+ Vincentt uses MediaPipe's face topology and top-left origin; it is not a
663
+ 106-point or bottom-left-compatible alias.
664
+
665
+ Run the example route `/face-info` to verify all landmarks, live bounds, nose
666
+ position, pose, eye distance, and `jawOpen` using `ScreenShape`, `ScreenText`,
667
+ and `Axes`.
668
+
669
+ ### `useFaceDetection()` — face presence lifecycle and count
670
+
671
+ Import `useFaceDetection` from `@vincentt-xr/sdk/tracking` when logic needs face
672
+ detected/lost edges, selected-face presence, or the current face count. It wraps
673
+ the shared `useFaceInfos` reader and does not start another model.
674
+
675
+ ```tsx
676
+ const detection = useFaceDetection({ faceIndex: 0, numFaces: 5 });
677
+
678
+ if (detection.justDetected) showEffect();
679
+ if (detection.detected) updateEffect(detection.face);
680
+ if (detection.justLost) hideEffect();
681
+ ```
682
+
683
+ | Visual-scripting term | SDK field |
684
+ | ------------------------------------- | ------------------------------------ |
685
+ | Face Detection Begin / face appeared | `justDetected`, `phase === "begin"` |
686
+ | Face Detection Stay / face present | `detected`, `phase === "stay"` |
687
+ | Face Detection End / face disappeared | `justLost`, `phase === "end"` |
688
+ | Face Detection None / no face | `!detected`, `phase === "none"` |
689
+ | Face Count / number of faces | `faceCount` and `faces.length` |
690
+ | Which Face / selected face | `faceIndex` option and `face` result |
691
+
692
+ The full result is `face`, `faces`, `faceCount`, `detected`, `justDetected`,
693
+ `justLost`, and `phase`. Options are the normal `useFaceInfos` options plus
694
+ `faceIndex`: `active`, `numFaces`, `targetFps`, `throttleMs`, and `holdMs`.
695
+ Default `holdMs: 250` prevents a transient missed frame from firing `end`; use
696
+ `holdMs: 0` only when immediate disappearance is required. Do not compare raw
697
+ result arrays or build a local previous-frame ref. The verified
698
+ `/face-detection` page tests five selectable face slots and all four phases; its
699
+ real-camera lifecycle flow passed on 26 July 2026.
700
+
701
+ ### Reusable normalized and face image crops
702
+
703
+ Import `drawNormalizedCrop` from `@vincentt-xr/sdk` for any normalized image
704
+ region. Import `drawFaceCrop` from `@vincentt-xr/sdk/tracking` when the bounds
705
+ come from `FaceInfo`. Both preserve the requested output pixel aspect ratio,
706
+ clamp at source edges, and draw into a caller-owned canvas without starting a
707
+ tracker or allocating per call.
708
+
709
+ ```ts
710
+ const cropCanvas = document.createElement("canvas");
711
+
712
+ const drawn = drawFaceCrop({
713
+ source: video,
714
+ face,
715
+ canvas: cropCanvas,
716
+ outputSize: { width: 224, height: 224 },
717
+ padding: 0.2,
718
+ });
719
+ ```
720
+
721
+ Bounds use normalized tracker-input/source coordinates with a top-left origin.
722
+ `padding` is the fraction of detected width/height added on each side. The face
723
+ helper defaults to `0.2`; the generic helper defaults to `0`. The functions do
724
+ not mirror, classify, upload, or retain pixels. A `false` result means the
725
+ source dimensions, bounds, output dimensions, or canvas context were unusable.
726
+ Reuse one canvas on frame-driven paths. Face attribute expression should feed
727
+ this crop into its independent image classifier; never derive emotion labels
728
+ from `FaceInfo.blendshapes`.
729
+
730
+ ### Face Actions — semantic face effect logic
731
+
732
+ Import `FACE_ACTION_NAMES`, `DEFAULT_FACE_ACTION_OPTIONS`,
733
+ `getFaceActionValue`, `hasFaceAction`, and `useFaceAction` from
734
+ `@vincentt-xr/sdk/tracking`. They derive from caller-owned `FaceInfo`; they do
735
+ not start a second tracker.
736
+
737
+ ```tsx
738
+ const face = useFaceInfo();
739
+ const blink = useFaceAction(face, "eyeBlink");
740
+
741
+ useEffect(() => {
742
+ if (blink.justStarted) capturePhoto();
743
+ }, [blink.justStarted]);
744
+
745
+ const mouthAmount = face ? getFaceActionValue(face, "mouthOpen") : 0;
746
+ ```
747
+
748
+ | Action | Source and direction | Default start / release / hold |
749
+ | ------------------------------------------------------------- | --------------------------------- | ------------------------------ |
750
+ | `browRaise` | max brow-up blendshape | `0.45 / 0.30 / 50 ms` |
751
+ | `eyeBlink`, `eyeBlinkLeft`, `eyeBlinkRight`, `eyeBlinkEither` | typed eye-blink blendshapes | `0.55 / 0.40 / 50 ms` |
752
+ | `mouthOpen` | `jawOpen` | `0.45 / 0.30 / 50 ms` |
753
+ | `mouthPout` | `mouthPucker` | `0.50 / 0.35 / 50 ms` |
754
+ | `headUp`, `headDown` | positive/negative pitch magnitude | `10° / 6° / 80 ms` |
755
+ | `headTurnLeft`, `headTurnRight` | positive/negative yaw magnitude | `10° / 6° / 80 ms` |
756
+ | `headTiltLeft`, `headTiltRight` | positive/negative roll magnitude | `10° / 6° / 80 ms` |
757
+
758
+ Blendshape values are `[0, 1]`; head values and thresholds are radians. Left
759
+ and up use the SDK-positive pose convention. Directional actions always return
760
+ a non-negative magnitude. Pass `neutralRotation: { ...face.pose.rotation }`
761
+ captured from an explicit user calibration gesture when the resting pose should
762
+ be treated as zero. Without it, head actions use zero rotation.
763
+
764
+ Use `getFaceActionValue` for continuous animation, `hasFaceAction` for a simple
765
+ level check, and `useFaceAction` for `active`, `justStarted`, `justEnded`, and
766
+ `phase: "begin" | "stay" | "end" | "none"`
767
+ with SDK hysteresis and hold timing. Do not guess blendshape strings, flip pose
768
+ signs, or rebuild local debounce/hysteresis logic. For multiple faces, select
769
+ one with `useFaceInfo({ faceIndex })`, or call
770
+ `getStrongestFaceAction(faces, action)` for an "any face" effect, then pass its
771
+ `face` into `useFaceAction`. The strongest selector is one pass and starts no
772
+ additional reader.
773
+
774
+ Effect House-style physical movement maps as follows: left/right/both/either
775
+ blink → `eyeBlinkLeft` / `eyeBlinkRight` / `eyeBlink` / `eyeBlinkEither`;
776
+ mouth open → `mouthOpen`; pout → `mouthPout`; eyebrow movement → `browRaise`.
777
+ Semantic blink left/right is viewer-facing and therefore swaps MediaPipe's raw
778
+ anatomical-side blendshape names. Never swap `FaceInfo.blendshapes` itself.
779
+ Every detected movement emits at least one `stay` frame after `begin` before
780
+ `end`.
781
+ These are observable movements, not image classification. Do not infer
782
+ `happy`, `sad`, `angry`, `fear`, `disgust`, `surprise`, or `neutral` from local
783
+ threshold combinations. Use the separate Face Attribute Expression API below.
784
+
785
+ Run `/face-actions` to inspect every live value, set neutral pose, tune the
786
+ selected action, and repeat the all-actions checklist. All actions and the
787
+ default thresholds were verified with the real-camera page on 26 July 2026.
788
+ Run the verified `/facial-movement-expression` page to test the Effect
789
+ House-compatible physical movement subset, phase edges, fixed-face or
790
+ strongest-face selection, and the image classifier. Its complete real-camera
791
+ flow passed on 26 July 2026.
792
+
793
+ ### Face Attribute Expression — image-based classification
794
+
795
+ Import `useFaceAttributeExpression`, `FACE_ATTRIBUTE_EXPRESSION_NAMES`, and
796
+ their types from `@vincentt-xr/sdk/tracking`. Pass caller-owned `FaceInfo`; the
797
+ hook reuses the shared session input frame, crops that face, and lazy-loads the
798
+ OpenCV MobileFaceNet INT8 ONNX classifier. It does not create another face
799
+ reader or detector.
800
+
801
+ ```tsx
802
+ const face = useFaceInfo();
803
+ const attribute = useFaceAttributeExpression(face, {
804
+ targetFps: 5,
805
+ minConfidence: 0.55,
806
+ });
807
+
808
+ if (attribute.status === "ready") {
809
+ const { expression, confidence, scores } = attribute.result;
810
+ }
811
+ ```
812
+
813
+ The lifecycle is `idle | loading | ready | error`. Ready results contain
814
+ `expression`, `confidence`, and `scores`. `expression` can be `angry`,
815
+ `disgust`, `fear`, `happy`, `neutral`, `sad`,
816
+ `surprise`, or `unknown`. `scores` contains only the seven real model classes;
817
+ there is no fabricated `unknown` score. `unknown` is the low-confidence fallback
818
+ below `minConfidence`. Default `targetFps` is `5`; the model is about 1.36 MB,
819
+ loads outside the main bundle, and is SHA-384 verified. ONNX Runtime's WASM
820
+ worker files load from the jsDelivr package path that matches the active
821
+ `onnxruntime-web` runtime version, not through the consuming app's bundler. The
822
+ runtime files are fetched from jsDelivr without SDK-managed SRI because ONNX
823
+ Runtime resolves its own WASM assets.
824
+
825
+ For best threaded SIMD performance, production pages that run expression
826
+ classification should be cross-origin isolated
827
+ (`Cross-Origin-Opener-Policy: same-origin` and
828
+ `Cross-Origin-Embedder-Policy: require-corp`) so `SharedArrayBuffer` is
829
+ available. Without those headers, ONNX Runtime falls back to single-threaded
830
+ WASM.
831
+
832
+ `modelUrl` overrides must accept float32 NCHW `[1, 3, 112, 112]`, with planar
833
+ RGB normalized to `[-1, 1]`, and return seven logits ordered `angry`, `disgust`,
834
+ `fear`, `happy`, `neutral`, `sad`, `surprise`. Do not assume an arbitrary
835
+ MobileFaceNet ONNX model is compatible. Failed loads are removed from the
836
+ session cache so a remounted consumer can retry.
837
+
838
+ This is a visual-expression estimate, not ground-truth emotion, intent,
839
+ identity, or mental state. Never replace it with blendshape heuristics. Run the
840
+ verified `/facial-movement-expression` page to inspect every live model score.
841
+
842
+ ### `<Axes>` — generic 3D orientation gizmo
843
+
844
+ Use `Axes` to visualize any 3D pose. X is red, Y is green, and Z is blue. Keep
845
+ tracking separate: bind a face, hand, or object pose through standard Three.js
846
+ `position` and `rotation` props.
847
+
848
+ ```tsx
849
+ import { Axes } from "@vincentt-xr/sdk";
850
+
851
+ <Axes size={0.38} position={pose.position} rotation={pose.rotation} />;
852
+ ```
853
+
252
854
  ### `<FaceTracker.Mesh />` / `<FaceMesh>` — Deforming 468-point face mesh
253
855
 
254
856
  For masks, makeup, and full-face decoration that deforms with skin.
@@ -353,6 +955,12 @@ Every tracker takes an `active` prop (default `true`). Set `active={false}` and
353
955
 
354
956
  When several consumers share one model, it runs while **any** of them is active. `active` gates whether the model runs; `perf.targetFps` (below) caps how often it runs while active — use both. `<Segmentation>` also takes `active` (distinct from its `visible` prop, which only gates mask exposure while inference keeps running).
355
957
 
958
+ Session lifecycle preserves that ownership state: `session.stop()` pauses every
959
+ tracker without releasing its MediaPipe resources, and the next `session.start()`
960
+ resumes only models that still have at least one active owner. Static-image
961
+ detection pulses use the same suspension path, so source switches do not
962
+ accidentally reactivate an `active={false}` tracker.
963
+
356
964
  ### `perf` — cap inference rate + camera resolution
357
965
 
358
966
  Set `perf` on `<XRProvider>` to trade tracking fidelity for lower load and heat. This is the single biggest lever if the device runs hot.
@@ -378,6 +986,11 @@ Per-tracker override beats the session default:
378
986
 
379
987
  The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — good for both dev laptops and end-user phones. Don't lower it further without a reason; don't remove it to "get full quality" unless you know the target device can take the heat.
380
988
 
989
+ SDK-owned tracker, occluder, and canonical-face assets are SHA-384 verified
990
+ against hashes pinned in the package. A missing hash for
991
+ `https://cdn.vincentt.studio` fails closed; custom consumer-owned asset URLs
992
+ remain best-effort and warn when no pinned hash exists.
993
+
381
994
  ---
382
995
 
383
996
  ## Reading tracker state
@@ -386,9 +999,11 @@ The base template ships `perf={{ targetFps: 24, camRes: 720 }}` by default — g
386
999
 
387
1000
  Wraps children; the child's world position is updated to the tracked landmark each frame. Use for all decoration that simply follows a face/hand/body point.
388
1001
 
389
- Valid targets:
1002
+ Valid targets are centralized in `FACE_TARGETS`; use that exported list for
1003
+ editor dropdowns instead of copying it. Left/right mean the tracked person's
1004
+ perspective.
390
1005
 
391
- - **Face:** `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`
1006
+ - **Face binding presets:** `face.center`, `face.forehead`, `face.noseTip`, `face.chin`, `face.leftEye`, `face.rightEye`, `face.leftEyeOuter`, `face.rightEyeOuter`, `face.leftCheek`, `face.rightCheek`, `face.mouthCenter`, `face.leftEarlobe`, `face.rightEarlobe`, `face.custom`
392
1007
  - **Hand:** `hand.wrist`, `hand.thumbTip`, `hand.indexTip`, `hand.middleTip`, `hand.ringTip`, `hand.pinkyTip`, `hand.indexBase`, `hand.middleBase`, `hand.ringBase`, `hand.pinkyBase`
393
1008
  - **Body:** `body.head`, `body.neck`, `body.leftShoulder`, `body.rightShoulder`, `body.leftElbow`, `body.rightElbow`, `body.leftWrist`, `body.rightWrist`, `body.leftHip`, `body.rightHip`, `body.leftKnee`, `body.rightKnee`, `body.leftAnkle`, `body.rightAnkle`
394
1009
 
@@ -403,6 +1018,32 @@ Valid targets:
403
1018
  </FaceTracker>
404
1019
  ```
405
1020
 
1021
+ `face.center` is the general default. Eye targets are composite eye centers;
1022
+ `*EyeOuter` targets are exact outer-corner landmarks. Earlobe targets are the
1023
+ nearest stable MediaPipe face-oval points, not anatomical earlobe detections.
1024
+ All Effect House `FaceBindingAnchorType` names therefore have a Vincentt
1025
+ equivalent, but their numeric canonical-mesh coordinates are not compatible.
1026
+
1027
+ For `face.custom`, pass normalized face-local coordinates. The origin is face
1028
+ center; X runs from person's left `-1` to right `+1`, Y from chin `-1` to
1029
+ forehead `+1`, and positive Z comes toward the camera.
1030
+
1031
+ ```tsx
1032
+ <FaceTracker>
1033
+ <TrackingAnchor target="face.custom" customAnchorPoint={{ x: 0.7, y: 0.2, z: 0 }} smoothing={15}>
1034
+ <mesh />
1035
+ </TrackingAnchor>
1036
+ </FaceTracker>
1037
+ ```
1038
+
1039
+ Scene objects use the same `TrackingAnchor` component settings. A 2D null with
1040
+ `ScreenTransform` + `TrackingAnchor` projects the preset into SDK design pixels,
1041
+ so all child Screen Shapes move as one group. Use `/face-binding-anchors` to
1042
+ test every preset on the editable `rose null`; the inspector target options
1043
+ come directly from `FACE_TARGETS`. Toggle `Follow Face Roll` to rotate the
1044
+ attachment with head tilt. The example hierarchy intentionally shows only the
1045
+ combined root object.
1046
+
406
1047
  **`smoothing` — smooth motion from throttled inference.** Inference is throttled (the `perf` API caps it at 24fps by default) while rendering runs at 60-120fps, so a raw anchor holds then jumps on each new sample — visible stepping under head/hand movement. A plain lerp only hides it at rest; under motion it still steps. Pass `smoothing` (`~15` is a good default) and the anchor runs a **One Euro filter** — the standard adaptive low-pass for interactive tracking: it smooths hard when the landmark moves slowly (kills jitter) and loosens as it speeds up (kills lag), and never extrapolates so it can't overshoot or wobble. Higher `smoothing` is snappier (less lag, slightly more jitter); lower is smoother. It snaps (resets the filter) on first placement and on re-acquire, so it never slides in from a stale position. `0` (default) keeps the raw snap. Rotation is not smoothed. Prefer this over a hand-rolled `useFrame` lerp — a hand-rolled lerp reintroduces the stepping.
407
1048
 
408
1049
  ```tsx
@@ -471,10 +1112,101 @@ import { GestureTracker, GestureTrigger } from "@vincentt-xr/sdk/tracking";
471
1112
 
472
1113
  In a v2-template project this is already mounted in `src/App.tsx` — do NOT add a second one there. Mentioned here because `customBackground` is how you do background replacement: pair it with `<Segmentation type="portrait" />` and pass the segmentation mask texture as `segmentationMask`.
473
1114
 
1115
+ `VideoBackground` is a flat camera-background component: its material keeps
1116
+ `depthTest={false}` and `depthWrite={false}`. Editor-style 3D viewports that
1117
+ treat the camera feed as a real scene plane should render a separate editor
1118
+ video plane instead of changing `VideoBackground` semantics.
1119
+
474
1120
  ---
475
1121
 
476
1122
  ## Mesh primitives and textures
477
1123
 
1124
+ `ScreenImage` keeps fitted clones synchronized for dynamic video and canvas textures.
1125
+
1126
+ ### Texture Types API usage
1127
+
1128
+ `ScreenImage` is the common render target for static and animated texture sources. Use
1129
+ the normal runtime import for rendering:
1130
+
1131
+ ```tsx
1132
+ import { ScreenImage, ScreenSpaceUI } from "@vincentt-xr/sdk";
1133
+
1134
+ <ScreenSpaceUI>
1135
+ <ScreenImage
1136
+ source={{ kind: "image", src: imageUrl }}
1137
+ materialType="unlit"
1138
+ stretchMode="fit"
1139
+ transparent
1140
+ />
1141
+ </ScreenSpaceUI>;
1142
+ ```
1143
+
1144
+ Supported source behavior:
1145
+
1146
+ - Image: static URL texture. No playback controls.
1147
+ - Runtime texture: pass `{ kind: "texture", texture }` when supplying an existing `THREE.Texture`.
1148
+ - Video: use a managed `THREE.VideoTexture`; support play/pause, restart, frame stepping, playback rate, forward, and reverse stepping. Negative `playbackRate` is not used.
1149
+ - GIF: decode frames into a canvas texture. Preserve GIF dimensions and per-frame delays before applying ScreenImage fitting.
1150
+ - Image sequence: draw loaded frames into a canvas texture using FPS and frame state.
1151
+
1152
+ Animated sources support forward, reverse, ping-pong, random, looping, finite loop
1153
+ counts, pause/resume at current frame, restart, frame status, and ended state where
1154
+ the source supports it. Static image sources must not expose animation controls.
1155
+
1156
+ ScreenImage inspector rules:
1157
+
1158
+ - `ScreenImageSettingsPanel` must read/write the same scene-object state used by the rendered `ScreenImage`.
1159
+ - Transform drag/place must be wired through `onScreenTransformChange`.
1160
+ - Use `mapScreenImageSettingsToProps` to convert inspector canvas coordinates into render props.
1161
+ - Do not pass inspector pixel dimensions directly as world-space `ScreenImage` dimensions.
1162
+ - Dynamic canvas/video texture updates must mark fitted texture clones for GPU upload.
1163
+
1164
+ Low-level texture integration imports from `@vincentt-xr/sdk/low-level`:
1165
+
1166
+ ```ts
1167
+ import {
1168
+ BasicAnimatedTextureController,
1169
+ createTextureSourceResolver,
1170
+ getPlaybackFrame,
1171
+ getSequenceFrame,
1172
+ type AnimatedTextureController,
1173
+ type AnimatedTexturePlayMode,
1174
+ type ResolvedTextureSource,
1175
+ type TextureSourceResolver,
1176
+ type TextureSourceProvider,
1177
+ } from "@vincentt-xr/sdk/low-level";
1178
+ ```
1179
+
1180
+ Controller lifecycle is: `play()`, `pause()`, `restart()`, optional `seek()`,
1181
+ `setPlayMode("forward" | "reverse" | "ping-pong" | "random")`,
1182
+ `setLoop(enabled, count)`, `subscribe(listener)`, then `dispose()`. Read
1183
+ `getSnapshot()` for state, frame/time, duration, frame count, loop, and errors;
1184
+ check `capabilities` before rendering a control.
1185
+
1186
+ Frame helpers are also available from the same `/low-level` door:
1187
+ `getPlaybackFrame(tick, frameCount, mode, loop, loopCount)` handles forward,
1188
+ reverse, ping-pong, random, loop, and ended math; `getSequenceFrame(elapsedSeconds,
1189
+ fps, frameCount)` maps elapsed time to a bounded sequence index.
1190
+
1191
+ Event compatibility: subscribe with `controller.on(type, listener)` and remove
1192
+ with the returned disposer or `off()`. Supported event types are
1193
+ `KeyFrameInSecond`, `Pause`, `PlayBegin`, `PlayEnd`, `Resume`, and `Stop`.
1194
+ Event snapshots include the current frame/time, playback state, loop settings,
1195
+ and optional duration/frame count. Use capability checks before exposing an
1196
+ event-driven control.
1197
+
1198
+ Consumer example: `docs/texture-api-consumer.md`.
1199
+
1200
+ Browser verification checklist: Chrome/Android (autoplay requires muted media),
1201
+ Safari/iOS (inline video requires `playsInline` and user-gesture fallback), and
1202
+ background tabs (timers may throttle; frame state must remain bounded). Validate
1203
+ source switching, pause-at-current-frame, loop-off ending, reverse/ping-pong,
1204
+ stretch modes, drag/place transforms, and unmount cleanup in each target.
1205
+
1206
+ `ResolvedTextureSource` owns a texture, optional controller, metadata, and
1207
+ ownership-safe `dispose()`. Abort stale loads, stop timers/RAF, remove media
1208
+ listeners, and never dispose externally owned textures.
1209
+
478
1210
  Use R3F intrinsic `<mesh>`, `<group>`, `<planeGeometry>`, `<meshBasicMaterial>`, etc. Load image textures with drei's `useTexture`.
479
1211
 
480
1212
  **Always name your meshes** (`name="myLabel"`). Names enable runtime lookup AND visual-feedback editing by the editor.
@@ -494,6 +1226,289 @@ const tex = useTexture("https://cdn.../badge.webp");
494
1226
 
495
1227
  ---
496
1228
 
1229
+ ## Scene-object runtime and authoring state
1230
+
1231
+ Runtime components import from `@vincentt-xr/sdk`. Scene-object data and store
1232
+ operations import from `@vincentt-xr/sdk/scene-object`. DOM editors import from
1233
+ `@vincentt-xr/sdk/debug-ui`.
1234
+
1235
+ ### `<Transform3D value={settings}>` and `<MeshRenderer value={settings}>`
1236
+
1237
+ `Transform3D` applies position, rotation (degrees), scale, visibility, and optional
1238
+ Drei transform controls to its children. `MeshRenderer` renders a cube, sphere,
1239
+ plane, or first-pass GLTF/GLB asset with basic/standard material settings.
1240
+
1241
+ ```tsx
1242
+ import {
1243
+ DEFAULT_MESH_RENDERER,
1244
+ DEFAULT_TRANSFORM_3D,
1245
+ MeshRenderer,
1246
+ Transform3D,
1247
+ } from "@vincentt-xr/sdk";
1248
+
1249
+ <Transform3D
1250
+ value={{
1251
+ ...DEFAULT_TRANSFORM_3D,
1252
+ position: { x: 0, y: 0, z: -1 },
1253
+ showTransformControls: true,
1254
+ }}
1255
+ onChange={setTransform}
1256
+ >
1257
+ <MeshRenderer
1258
+ value={{
1259
+ ...DEFAULT_MESH_RENDERER,
1260
+ material: { ...DEFAULT_MESH_RENDERER.material, color: "#38bdf8" },
1261
+ }}
1262
+ />
1263
+ </Transform3D>;
1264
+ ```
1265
+
1266
+ `Transform3DProps` is `{ value, onChange?, children }`.
1267
+ `MeshRendererProps` is `{ value, renderOrder? }`. `renderOrder` is applied to the
1268
+ renderable mesh itself. Disabled or invisible settings render nothing.
1269
+
1270
+ `MeshRendererSettings` fields:
1271
+
1272
+ - `enabled`, `visible`: gate rendering.
1273
+ - `mesh`: `"cube" | "sphere" | "plane"` for primitive fallback.
1274
+ - `assetUrl?`: optional GLTF/GLB URL. When present, the asset renders instead of
1275
+ the primitive mesh.
1276
+ - `assetNodeName?`: optional object name inside the loaded GLTF scene. When
1277
+ missing or unresolved, the whole cloned scene renders.
1278
+ - `material.type`: `"standard" | "basic"`.
1279
+ - `material.color`, `opacity`, `wireframe`, `roughness`, `metalness`: material
1280
+ overrides. Wireframe belongs to material state, not geometry state.
1281
+ - `material.renderState?`: optional advanced draw behavior:
1282
+ - `depthTest`, `depthWrite`, `colorWrite`: booleans. `colorWrite` is the
1283
+ all-channel write gate.
1284
+ - `colorMask`: `{ r, g, b, a }` per-channel color write gates. This is
1285
+ applied as scoped WebGL color-mask state for each mesh draw and restored
1286
+ after the object renders.
1287
+ - `depthFunc`: `"never" | "less" | "equal" | "lessEqual" | "greater" |
1288
+ "notEqual" | "greaterEqual" | "always"`.
1289
+ - `cullMode`: `"back" | "front" | "off"`. The default `"back"` maps to
1290
+ Three.js `FrontSide`; `"off"` maps to `DoubleSide`.
1291
+ - `blendMode`: `"normal" | "additive" | "subtractive" | "multiply" |
1292
+ "none"`.
1293
+ - `polygonOffset`: `{ enabled, factor, units }`.
1294
+ - `shadowMode`: `"off" | "cast" | "receive" | "castReceive"`.
1295
+
1296
+ Current GLTF support is intentionally first-pass: the renderer clones the loaded
1297
+ scene and applies color, opacity, wireframe, render-state fields, and shadow
1298
+ flags to cloned mesh materials.
1299
+ Do not rely on it yet for advanced material graph preservation, animation,
1300
+ skinning behavior, or production asset lifecycle guarantees.
1301
+
1302
+ Render-state defaults preserve current behavior: depth test/write on,
1303
+ `lessEqual` depth function, color write on, back-face culling, normal blending,
1304
+ RGBA color mask fully enabled, and polygon offset disabled. Instance count is not
1305
+ part of `MeshRenderer.material.renderState`; it belongs to instanced mesh
1306
+ rendering rather than material shading state.
1307
+
1308
+ `assetUrl` uses Drei `useGLTF`, so it suspends while loading. `MeshRenderer`
1309
+ wraps asset-backed rendering in `Suspense fallback={null}` so data-driven
1310
+ `SceneObjectRenderer` usage is safe by default. Add your own outer `Suspense`
1311
+ or call `useGLTF.preload(url)` only when you want a visible loading state or
1312
+ early fetch. Primitive meshes do not require Suspense.
1313
+
1314
+ ### Screen Transform data and resolver
1315
+
1316
+ Import `ScreenTransform2DSettings`, `DEFAULT_SCREEN_TRANSFORM_2D`,
1317
+ `DEFAULT_SCREEN_TRANSFORM_ROOT`, `DEFAULT_REFERENCE_PIXELS_PER_UNIT`,
1318
+ `mergeScreenTransform2D`, and `resolveScreenTransform2D` from the core door.
1319
+
1320
+ `ScreenTransform2DSettings` owns enabled/visible state, canvas-pixel position and
1321
+ size, normalized pivot, degree rotation, 2D scale, reference pixels per unit,
1322
+ render order, overlay mode, and guide visibility.
1323
+
1324
+ ```ts
1325
+ import {
1326
+ DEFAULT_SCREEN_TRANSFORM_2D,
1327
+ mergeScreenTransform2D,
1328
+ resolveScreenTransform2D,
1329
+ type ScreenTransform2DSettings,
1330
+ } from "@vincentt-xr/sdk";
1331
+
1332
+ const transform: ScreenTransform2DSettings = mergeScreenTransform2D({
1333
+ position: { x: 0, y: 240 },
1334
+ size: { width: 420, height: 420 },
1335
+ showTransformGuides: true,
1336
+ });
1337
+
1338
+ const resolved = resolveScreenTransform2D({
1339
+ transform,
1340
+ parentSize: { width: 9, height: 16 },
1341
+ });
1342
+ ```
1343
+
1344
+ Use `mergeScreenTransform2D` to fill nested defaults. Use
1345
+ `resolveScreenTransform2D` whenever runtime or editor tooling must convert the
1346
+ authoring model into center, size, rotation, pivot offset, and aspect. Do not copy
1347
+ the default `720x1280` root or `32` reference-pixel constant into another renderer
1348
+ or guide.
1349
+
1350
+ ### Screen/world coordinate utilities
1351
+
1352
+ Import the `*ScreenPixels` coordinate helpers, `screenPointToRay`, and
1353
+ `smoothNumber` / `smoothVector2` / `smoothVector3` from `@vincentt-xr/sdk`. Use
1354
+ these shared helpers instead of rewriting mapping or projection math or assuming
1355
+ design pixels are Three.js world units.
1356
+ SDK screen pixels use the Effect House-style `720x1280` design canvas, center
1357
+ origin, and +Y up.
1358
+
1359
+ ```ts
1360
+ const units = screenPixelsToUnits(120, { parentSize: screenDimensions });
1361
+ const pixels = screenUnitsToPixels(units, { parentSize: screenDimensions });
1362
+
1363
+ const labelPosition = worldToScreenPixels({
1364
+ point: trackedObject.getWorldPosition(new THREE.Vector3()),
1365
+ camera,
1366
+ viewportSize,
1367
+ });
1368
+
1369
+ const placementRay = screenPointToRay({
1370
+ point: { x: tapX, y: tapY },
1371
+ camera,
1372
+ viewportSize,
1373
+ });
1374
+ ```
1375
+
1376
+ - Use pixel/unit conversion for Screen Image, Screen Text, Screen Shape, editor
1377
+ handles, and custom `ScreenSpaceUI` content.
1378
+ - Use `worldToScreenPixels` for labels, prompts, and indicators that follow a
1379
+ face, hand, body joint, or 3D object. Its `depth` is projected NDC depth; hide
1380
+ the result when `depth` is outside `[-1, 1]` because it is outside the camera
1381
+ clip range.
1382
+ - Use `screenPointToRay` for tap selection, hit testing, and 3D placement. Pass
1383
+ design-canvas coordinates, not DOM client pixels.
1384
+ - Use `clientToScreenPixels` / `screenPixelsToClient` for pointer events and DOM
1385
+ overlays; pass the viewport element's `getBoundingClientRect()` result.
1386
+ - Use `normalizedToScreenPixels` / `screenPixelsToNormalized` for normalized
1387
+ viewport coordinates with top-left origin. These values are not automatically
1388
+ camera-crop adjusted.
1389
+ - Use `mediaToScreenPixels` / `screenPixelsToMedia` with the shared
1390
+ `XRMediaLayout` for camera/image pixels. They apply cover/contain offsets,
1391
+ scaling, and optional display mirroring; do not manually repeat that math.
1392
+ - Use `smoothNumber`, `smoothVector2`, or `smoothVector3` for tracked animation.
1393
+ Pass the R3F frame `delta` in seconds and a smoothing rate (larger is faster).
1394
+ Reuse the optional `out` vector in per-frame paths to avoid allocations.
1395
+ - Zero-size viewport/parent inputs return neutral zero coordinates or units so
1396
+ first paint and collapsed layouts do not throw. Invalid custom root sizes and
1397
+ `referencePixelsPerUnit` still throw because they are configuration errors.
1398
+ - Use Three.js `localToWorld`, `worldToLocal`, `Vector3.project/unproject`,
1399
+ `set`, `toArray`, and `fromArray` directly; do not add SDK wrappers for them.
1400
+
1401
+ ### Scene-object store
1402
+
1403
+ ```ts
1404
+ import {
1405
+ addComponentToObject,
1406
+ createDefaultComponent,
1407
+ createSceneObject,
1408
+ deleteSceneObject,
1409
+ getOrderedComponents,
1410
+ moveComponent,
1411
+ removeComponent,
1412
+ type SceneObjectStore,
1413
+ } from "@vincentt-xr/sdk/scene-object";
1414
+
1415
+ const cube = createSceneObject({ name: "Cube" });
1416
+ const transform = createDefaultComponent("Transform", cube.id);
1417
+ const mesh = createDefaultComponent("MeshRenderer", cube.id);
1418
+ cube.componentIds = [transform.id, mesh.id];
1419
+
1420
+ const store: SceneObjectStore = {
1421
+ objects: { [cube.id]: cube },
1422
+ components: { [transform.id]: transform, [mesh.id]: mesh },
1423
+ };
1424
+
1425
+ const ordered = getOrderedComponents(store, cube.id);
1426
+ ```
1427
+
1428
+ `SceneObject` owns identity, display name, enabled state, optional `renderGroupId`,
1429
+ optional hierarchy fields, and ordered `componentIds`. `SceneComponent` owns one
1430
+ typed settings payload and points back to its object. `ComponentRegistry` maps
1431
+ component types to those settings.
1432
+
1433
+ Persisted scenes may still provide the deprecated `layerId`; migration treats it
1434
+ as a fallback when `renderGroupId` is absent. New code should only write
1435
+ `renderGroupId`.
1436
+
1437
+ Nested scene-object fields:
1438
+
1439
+ - `parentObjectId?: SceneObjectId | null` points to the parent object.
1440
+ - `childObjectIds?: SceneObjectId[]` stores ordered child IDs for hierarchy and
1441
+ runtime recursion.
1442
+
1443
+ Nested transforms are evaluated through the runtime object tree. Child objects
1444
+ render inside the parent object, so parent movement affects descendants while
1445
+ each child keeps its own local component settings.
1446
+
1447
+ Store helpers are immutable and preserve extra fields on an app-extended store:
1448
+
1449
+ - `addComponentToObject(store, component)` adds the component and its ID.
1450
+ - `moveComponent(store, objectId, componentId, nextIndex)` reorders one object.
1451
+ - `removeComponent(...)` removes the component and its ID.
1452
+ - `deleteSceneObject(store, objectId)` removes the object and all owned components.
1453
+ - `replaceSceneComponent(store, component)` replaces one component.
1454
+ - `replaceSceneObject(store, object)` replaces one object.
1455
+ - `setSceneObjectVisible(store, objectId, visible)` toggles object `enabled`.
1456
+ - `getSceneObjectChildren(store, objectId)` resolves explicit and implicit child
1457
+ objects in hierarchy order.
1458
+ - `setSceneObjectParent(store, objectId, parentObjectId, nextIndex?)` updates both
1459
+ sides of a parent/child relationship and rejects cycles or mixed transform
1460
+ domains. Parent `Transform` under `Transform`, and `ScreenTransform` under
1461
+ `ScreenTransform`. Existing child values become local-to-parent values.
1462
+ - `setSceneObjectParentPreserveWorldTransform(store, objectId, parentObjectId,
1463
+ nextIndex?)` is the editor seam for drag/drop and parent dropdowns. It updates
1464
+ the hierarchy and recalculates the child's local `Transform` or
1465
+ `ScreenTransform` so the object does not visually jump when nested or
1466
+ un-nested. This is exact for representable transforms. Rotation combined with
1467
+ non-uniform ancestor scale introduces shear, which neither component stores,
1468
+ so preservation in that case is approximate.
1469
+ - `createOrderedComponentsSelector()` creates a memoized ordered selector.
1470
+
1471
+ ### `<SceneObjectRenderer>`
1472
+
1473
+ `SceneObjectRenderer` reads an object plus its ordered components and renders the
1474
+ supported composition. Import the renderer from core and its store type from
1475
+ `/scene-object`.
1476
+
1477
+ ```tsx
1478
+ import { SceneObjectRenderer } from "@vincentt-xr/sdk";
1479
+ import type { SceneComponent, SceneObjectStore } from "@vincentt-xr/sdk/scene-object";
1480
+
1481
+ <SceneObjectRenderer
1482
+ objectId={cube.id}
1483
+ store={store}
1484
+ editorMode
1485
+ onComponentChange={(component: SceneComponent) => replaceComponent(component)}
1486
+ />;
1487
+ ```
1488
+
1489
+ Props are `objectId`, `store`, optional `editorMode`, optional
1490
+ `onComponentChange`, and optional `fallback`. Supported compositions are:
1491
+
1492
+ - `Transform` + `MeshRenderer`
1493
+ - `Transform` + `Light`
1494
+ - `Transform` + `MeshRenderer` with optional `LookAt` and `PinToMesh`
1495
+ - `Transform` only as an invisible 3D null/container
1496
+ - `ScreenTransform` only as an invisible 2D null/container
1497
+ - `ScreenTransform` + `ScreenImage`
1498
+ - `ScreenTransform` + `ScreenText`
1499
+
1500
+ For mesh objects, `SceneObjectRenderer` registers the object id at runtime so
1501
+ behavior components can resolve serialized target ids. Inspector/editor changes
1502
+ must route through `onComponentChange` and replace the selected object's
1503
+ component in the store; do not keep separate page-local copies of transform,
1504
+ material, light, LookAt, or PinToMesh state.
1505
+
1506
+ In editor mode, transform changes return through `onComponentChange`.
1507
+ Unsupported compositions render nothing in production and use `fallback` in
1508
+ editor mode.
1509
+
1510
+ ---
1511
+
497
1512
  ## `<ScreenText>` — Standard screen-space text
498
1513
 
499
1514
  Import from `@vincentt-xr/sdk`. Use this when you want the standard
@@ -545,6 +1560,387 @@ Use `TextLabel` when:
545
1560
  - you want to place text manually in world space or inside a custom transform
546
1561
  - you are building a lower-level composite where another component owns layout
547
1562
 
1563
+ `SceneObjectRenderer` owns the required `ScreenSpaceUI` container when it renders
1564
+ an object composed from `ScreenTransform` + `ScreenText` or `ScreenImage`. Do not
1565
+ wrap those renderer instances in an additional `ScreenSpaceUI`; direct uses of
1566
+ `ScreenText` and `ScreenImage` still require the container shown above.
1567
+
1568
+ Debug panels are independent, selection-driven surfaces. Use `HierarchyPanel` for a
1569
+ render-group-aware scene-object list and render one
1570
+ `ObjectInspectorPanel` for the selected object. Both accept `initialPosition` so the
1571
+ hierarchy and Inspector can be presented side by side instead of overlapping.
1572
+ `ScreenTextSettingsPanel` and `ScreenImageSettingsPanel` already own their complete
1573
+ Inspector UI; pass their optional `inspector` binding when they edit an object from
1574
+ a shared scene store. Do not nest either one inside another `ObjectInspectorPanel`.
1575
+ `ObjectInspectorPanel` is uncontrolled by default, but multi-object editors can pass
1576
+ `position` and `onPositionChange` to preserve one Inspector location while swapping
1577
+ the selected object's panel. `SceneObjectRenderer` forwards screen transform-guide
1578
+ drags through `onComponentChange`; selection policies such as closing active transform
1579
+ controls belong to the parent editor, not to `HierarchyPanel` or the standalone panels.
1580
+
1581
+ ### Reusable XR scene editor debug UI
1582
+
1583
+ For temporary client-app scene setup, prefer the reusable scene editor adapters
1584
+ from `@vincentt-xr/sdk/debug-ui` instead of rewriting viewport, hierarchy, and
1585
+ inspector wiring in each app. These are debug/authoring adapters over the
1586
+ scene-object runtime, not production runtime UI.
1587
+
1588
+ ```tsx
1589
+ import {
1590
+ SceneHierarchyPanel,
1591
+ SceneInspectorPanel,
1592
+ ViewportToolbar,
1593
+ XRSceneEditorProvider,
1594
+ XRSceneViewport,
1595
+ } from "@vincentt-xr/sdk/debug-ui";
1596
+ import {
1597
+ replaceSceneComponent,
1598
+ replaceSceneObject,
1599
+ type RenderGroupSceneObjectStore,
1600
+ } from "@vincentt-xr/sdk/scene-object";
1601
+
1602
+ <XRSceneEditorProvider
1603
+ store={store}
1604
+ selectedObjectId={selectedObjectId}
1605
+ onSelectObject={setSelectedObjectId}
1606
+ onStoreChange={setStore}
1607
+ onComponentChange={(component) =>
1608
+ setStore((current) => replaceSceneComponent(current, component))
1609
+ }
1610
+ onObjectChange={(object) => setStore((current) => replaceSceneObject(current, object))}
1611
+ >
1612
+ <XRSceneViewport />
1613
+ <ViewportToolbar />
1614
+ <SceneHierarchyPanel />
1615
+ <SceneInspectorPanel />
1616
+ </XRSceneEditorProvider>;
1617
+ ```
1618
+
1619
+ All pieces are optional and composable:
1620
+
1621
+ - `XRSceneViewport` renders `XRScene`, an editor-owned camera-texture plane,
1622
+ editor grid, camera, orbit controls, and `SceneObjectRenderer` entries from
1623
+ `createRenderPlan`. The editor video plane consumes the same camera texture as
1624
+ `VideoBackground`, but it is a separate debug-ui plane with `depthTest` and
1625
+ `depthWrite` enabled. Do not add depth props to core `VideoBackground`.
1626
+ - `SceneHierarchyPanel` adapts `HierarchyPanel` to a
1627
+ `RenderGroupSceneObjectStore`.
1628
+ - `SceneInspectorPanel` renders component settings for the selected object and
1629
+ routes edits through `onComponentChange` or `onObjectChange`.
1630
+ - `ViewportToolbar` owns transient viewport controls such as orbit on/off.
1631
+
1632
+ Keep `SceneObjectStore`/`RenderGroupSceneObjectStore` as the only persistent
1633
+ scene data. Selection, orbit state, grid visibility, video-background visibility,
1634
+ and editor camera choices are transient editor session state. Do not keep
1635
+ page-local shadow copies of Transform, MeshRenderer, Light, LookAt, or PinToMesh
1636
+ settings.
1637
+
1638
+ Use these adapters in development/debug surfaces such as `/viewport-editor`
1639
+ or a temporary client-app editor. Omit them for final preview/runtime and render
1640
+ the scene through core runtime components instead.
1641
+
1642
+ For debug pages that need a camera/image/video picker, import
1643
+ `MediaSourceSwitcher` and `useFaceTrackingMediaSourceSession` from
1644
+ `@vincentt-xr/sdk/debug-ui`. `useFaceTrackingMediaSourceSession(activeMediaSource)`
1645
+ applies the selected debug media preset to the current XR session and starts
1646
+ the face-tracking media pipeline used by the debug face examples. The older
1647
+ `useMediaSourceSession(activeMediaSource, { pipeline })` name remains available
1648
+ for custom debug pipelines. Use these only in debug/example UI; production
1649
+ runtime should set the session media source directly.
1650
+
1651
+ ### Debug UI usage in Harness client apps
1652
+
1653
+ #### AI-agent integration contract
1654
+
1655
+ Choose the import door by responsibility. Do not search other doors for an
1656
+ equivalent symbol and do not import SDK source files directly.
1657
+
1658
+ | Agent task | Import door | Symbols |
1659
+ | ----------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------ |
1660
+ | Render managed objects and compose passes | `@vincentt-xr/sdk` | `SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer` |
1661
+ | Guard internal render layers | `@vincentt-xr/sdk` | `MAX_RENDER_GROUP_PASSES`, `getRenderGroupLayer`, `EXTERNAL_BACKGROUND_RENDER_LAYER` |
1662
+ | Create and mutate scene-object state | `@vincentt-xr/sdk/scene-object` | `createSceneObject`, `createDefaultComponent`, store helpers |
1663
+ | Materialize and order render groups | `@vincentt-xr/sdk/scene-object` | `migrateToRenderGroups`, `createRenderPlan`, reorder/move helpers |
1664
+ | Render DOM authoring controls | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`, settings panels |
1665
+ | Type a bulk hierarchy export | `@vincentt-xr/sdk/debug-ui` | `HierarchyPanelCopyConfig` |
1666
+
1667
+ The integrating client, not the SDK panels, must own all mutable editor state:
1668
+
1669
+ - the complete scene store, including `objects`, `components`, `renderGroups`,
1670
+ and `renderGroupOrder`;
1671
+ - `selectedObjectId` and which selected Inspector is rendered;
1672
+ - component replacement and object/group visibility updates;
1673
+ - reorder and compatibility-checked move operations;
1674
+ - the rule that changing selection closes transform controls and guides belonging
1675
+ to the previously selected object; and
1676
+ - the `HierarchyPanelCopyConfig` value.
1677
+
1678
+ Required invariants:
1679
+
1680
+ - Mount DOM debug panels outside `XRScene`; mount R3F/Three.js renderers and
1681
+ controls inside `XRScene`.
1682
+ - Keep both branches below the same parent state and inside `XRProvider`.
1683
+ - Derive group order from `scene.renderGroupOrder`; never hardcode a canonical
1684
+ order such as background/general/tracking/foreground/post.
1685
+ - Use `SequentialRenderPasses` for guaranteed cross-group composition. Do not
1686
+ replace it with a larger Three.js `renderOrder` value.
1687
+ - Treat camera layers as internal visibility masks, not user-facing layer order.
1688
+ - Keep each managed object in exactly one materialized group and exactly once in
1689
+ that group's `objectIds`.
1690
+ - Do not show transform controls for an unselected object. Initial control state
1691
+ must agree with the initial `selectedObjectId`.
1692
+ - Do not create more than `MAX_RENDER_GROUP_PASSES` managed groups.
1693
+ - Treat Copy Config as a versioned export/debug payload, not as an import or
1694
+ persistence API.
1695
+
1696
+ Use these state transitions:
1697
+
1698
+ | Event | Required parent-state update |
1699
+ | ----------------- | -------------------------------------------------------------------------------- |
1700
+ | Select object | Set `selectedObjectId`; disable all existing 2D guides and 3D transform controls |
1701
+ | Edit component | Replace that component immutably in `scene.components` |
1702
+ | Reorder group | Call `reorderRenderGroup`; keep object lists unchanged |
1703
+ | Reorder object | Call `reorderObjectWithinGroup`; change only the selected group's `objectIds` |
1704
+ | Move object | Validate compatibility, then call `moveObjectToGroup` |
1705
+ | Toggle visibility | Update the object or group; do not encode visibility in selection state |
1706
+ | Render frame | Derive `createRenderPlan(scene)` and active pass layers from current state |
1707
+
1708
+ Failure recovery for generated integrations:
1709
+
1710
+ | Symptom | Likely cause | Required fix | Do not |
1711
+ | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- |
1712
+ | “R3F hooks can only be used within Canvas” | R3F renderer/control mounted in the DOM branch, or duplicate React/R3F packages | Move the scene component inside `XRScene`; in link mode pin React, React DOM, Three.js, R3F, and Drei to client copies | Move DOM panels into `XRScene` |
1713
+ | Hierarchy order changes but visual composition does not | Only `renderOrder` or array order changed | Recompute the render plan and pass the ordered layers to `SequentialRenderPasses` | Expose camera-layer numbers as visual order |
1714
+ | Wrong object's gizmo remains visible | Selection changed without clearing component control flags | Disable all stale transform controls/guides in the same selection transition | Assume `HierarchyPanel` owns control state |
1715
+ | Screen transform drag stops after one move | An unstable callback caused active pointer listeners to be replaced | Use current SDK components and keep component replacement callbacks wired to parent state | Fork private drag logic into the DOM panel |
1716
+ | Cross-group drop silently fails | Target policy rejects the object's component composition | Call `validateGroupCompatibility` and preserve the original store on `false` | Force incompatible membership |
1717
+ | Group is omitted from the render plan | Its zero-based index is `>= MAX_RENDER_GROUP_PASSES` (`30`), so the 31st group onward is beyond the managed-pass limit | Prevent creation using `MAX_RENDER_GROUP_PASSES` | Reuse/clamp multiple groups onto one layer |
1718
+ | Copied JSON cannot be restored | Copy Config has no public importer | Treat it as export/debug handoff only | Invent an undocumented restore API |
1719
+
1720
+ Harness clients must keep the DOM authoring UI and R3F scene integration in sibling
1721
+ branches under the same parent-owned state:
1722
+
1723
+ ```text
1724
+ XRProvider
1725
+ └─ AspectRatioContainer
1726
+ ├─ XRScene (R3F Canvas)
1727
+ │ ├─ camera, lights, and scene content
1728
+ │ ├─ SequentialRenderPasses
1729
+ │ ├─ SceneObjectRenderer instances
1730
+ │ └─ transform guides and controls
1731
+ ├─ HierarchyPanel
1732
+ └─ selected-object Inspector or standalone settings panel
1733
+ ```
1734
+
1735
+ Mount `HierarchyPanel`, `ObjectInspectorPanel`, `ComponentEditor`,
1736
+ `ScreenTextSettingsPanel`, and `ScreenImageSettingsPanel` outside `XRScene`.
1737
+ They render DOM portals and communicate through props and callbacks. Mount
1738
+ `SceneObjectRenderer`, `SequentialRenderPasses`, `ScreenSpaceUI`, `Transform3D`,
1739
+ and all Three.js content inside `XRScene` because they use R3F Canvas context.
1740
+ Keep both branches inside `XRProvider`.
1741
+
1742
+ The parent Harness component owns:
1743
+
1744
+ - the complete scene-object store;
1745
+ - `selectedObjectId`;
1746
+ - component replacement;
1747
+ - group and object reorder callbacks;
1748
+ - visibility callbacks;
1749
+ - compatibility-checked moves; and
1750
+ - the bulk Copy Config value.
1751
+
1752
+ The data loop is:
1753
+
1754
+ ```text
1755
+ Hierarchy / Inspector callback
1756
+ ↓
1757
+ parent scene store
1758
+ ↓
1759
+ selection + render plan + active pass layers
1760
+ ↓
1761
+ SceneObjectRenderer inside XRScene
1762
+ ↓
1763
+ transform-control change
1764
+ └──────────────→ parent scene store
1765
+ ```
1766
+
1767
+ If React reports that R3F hooks can only be used within Canvas, a scene renderer,
1768
+ render-pass controller, transform guide, or Three.js control has crossed into the
1769
+ DOM panel branch. Do not solve this by moving the panels into `XRScene`; move the
1770
+ R3F-dependent scene component back inside `XRScene` and keep the shared state above
1771
+ both branches.
1772
+
1773
+ The working `test001` Harness client is the canonical external integration. It runs
1774
+ a Cube, Screen Image, and Screen Text together with `HierarchyPanel`,
1775
+ selection-driven Inspectors, component editors, group/object ordering, visibility,
1776
+ transform controls, sequential render passes, and bulk Copy Config. The SDK
1777
+ `/layer-panel-test` page is the matching in-repo fixture.
1778
+
1779
+ For SDK co-development, run `pnpm build:watch` in `vincentt-xr-sdk` and
1780
+ `SDK_LINK=1 pnpm dev` in the Harness client. Use a different `PORT` when the SDK
1781
+ example is already using 5173, for example
1782
+ `PORT=5174 SDK_LINK=1 pnpm dev`. Also validate the client typecheck and linked
1783
+ production build; the in-repo SDK example alone does not validate package exports
1784
+ or Canvas placement in a separate client.
1785
+
1786
+ ### Render groups and hierarchy ordering
1787
+
1788
+ Import render-group types, policies, migration, compatibility, reorder helpers, and
1789
+ `createRenderPlan` from `@vincentt-xr/sdk/scene-object`. Import
1790
+ `SceneObjectRenderer`, `SequentialRenderPasses`, `RenderGroupLayer`,
1791
+ `getRenderGroupLayer`, and `EXTERNAL_BACKGROUND_RENDER_LAYER` from core.
1792
+
1793
+ Use `migrateToRenderGroups(store, objectOrder)` to group a flat scene-object store.
1794
+ It preserves compatible `renderGroupId` assignments, infers missing or incompatible
1795
+ assignments from components, and creates only the groups required by actual scene
1796
+ objects. Their initial order follows the first occurrence in `objectOrder`; there is
1797
+ no fixed render-order list. New
1798
+ objects can call `getDefaultRenderGroupId(components)` after their components are
1799
+ assembled.
1800
+
1801
+ The ownership boundary is:
1802
+
1803
+ | Concern | Owner |
1804
+ | -------------------------------------------------- | -------------------------- |
1805
+ | Object identity, display name, and component order | `SceneObject` |
1806
+ | Object behavior and settings | `SceneComponent` |
1807
+ | Cross-domain composition and group visibility | `RenderGroup` |
1808
+ | Order inside a group | `RenderGroup.objectIds` |
1809
+ | Order between groups | `renderGroupOrder` |
1810
+ | Structural commands | `HierarchyPanel` callbacks |
1811
+ | Property editing and selection policy | Parent editor + Inspector |
1812
+ | Object-to-pass mapping | `createRenderPlan()` |
1813
+ | Final frame composition | `SequentialRenderPasses` |
1814
+
1815
+ The Hierarchy and Inspector emit commands but do not own scene state or rendering.
1816
+ Every managed object must belong to exactly one materialized group, appear exactly
1817
+ once across all group `objectIds` arrays, and have a matching `renderGroupId`.
1818
+ Selection is editor state and never changes render order.
1819
+
1820
+ Use only the immutable helpers to change structure:
1821
+
1822
+ - `reorderRenderGroup(store, groupId, nextIndex)` changes group composition order.
1823
+ - `reorderObjectWithinGroup(store, groupId, objectId, nextIndex)` sorts within a group.
1824
+ - `moveObjectToGroup(...)` moves only when `validateGroupCompatibility(...)` passes.
1825
+ - `createRenderPlan(store)` returns top-first entries and stable 1000-value sorting bands.
1826
+
1827
+ Unknown group/object IDs, incompatible targets, and reorders to the current index
1828
+ are safe no-ops that return the original store. `validateGroupCompatibility` is
1829
+ read-only and returns `false` for missing objects, missing groups, or rejected
1830
+ component compositions.
1831
+
1832
+ Pass each plan entry's `renderOrder` and `renderLayer` to `SceneObjectRenderer`.
1833
+ Mount one `SequentialRenderPasses` with the active plan layers in top-first order.
1834
+ It takes over the R3F loop, preserves color between groups, and clears depth before
1835
+ each later group. Bottom groups therefore composite in front even across Three.js's
1836
+ opaque and transparent queues. `captureFrame()` uses the same grouped renderer.
1837
+
1838
+ These mechanisms are different:
1839
+
1840
+ - `renderGroupOrder` controls cross-group sequential composition. The first group
1841
+ renders first; the last group renders last and appears in front.
1842
+ - `RenderGroup.objectIds` controls order inside one group.
1843
+ - Three.js camera layers are internal visibility masks; their numeric values do not
1844
+ define visual order.
1845
+ - `renderOrder` is a sorting value inside one render call. It cannot by itself
1846
+ impose ordering across opaque and transparent queues.
1847
+
1848
+ ```tsx
1849
+ const plan = createRenderPlan(scene);
1850
+ const passLayers = scene.renderGroupOrder.flatMap((groupId, groupIndex) =>
1851
+ scene.renderGroups[groupId]?.enabled && scene.renderGroups[groupId].objectIds.length
1852
+ ? [getRenderGroupLayer(groupIndex)]
1853
+ : [],
1854
+ );
1855
+
1856
+ <SequentialRenderPasses layers={passLayers} />;
1857
+ {
1858
+ plan.map((entry) => (
1859
+ <SceneObjectRenderer
1860
+ key={entry.objectId}
1861
+ objectId={entry.objectId}
1862
+ store={scene}
1863
+ renderOrder={entry.renderOrder}
1864
+ renderLayer={entry.renderLayer}
1865
+ />
1866
+ ));
1867
+ }
1868
+ ```
1869
+
1870
+ Layer 0 is shared across passes for lights and non-renderable scene infrastructure.
1871
+ All visible managed project content in a grouped scene must be assigned to a group
1872
+ layer. `SequentialRenderPasses` also enables its layers on the R3F interaction
1873
+ raycaster. `RenderGroupLayer` leaves Drei `TransformControls` on shared layer 0,
1874
+ so 2D guides and 3D move/rotate/scale gizmos remain draggable after render-group
1875
+ isolation.
1876
+
1877
+ For legacy flat camera backgrounds, render `VideoBackground` outside managed
1878
+ scene-object content. For the reusable scene editor, use `XRSceneViewport`; it
1879
+ owns a separate depth-writing camera-texture plane for editor-only 3D placement.
1880
+
1881
+ `HierarchyPanel` receives ordered `groups`, scene `objects`, selection callbacks,
1882
+ and separate callbacks for group reorder, within-group reorder, and explicit moves.
1883
+ It owns collapse UI and drop feedback, but the parent store owns selection, group
1884
+ visibility, compatibility validation, the Inspector, and rendering.
1885
+ Empty reserved groups are not materialized in the store or presented in the hierarchy.
1886
+ The panel is expanded by default and can fold to its title button without losing
1887
+ its position, current selection, or per-group collapse state. The normal hierarchy
1888
+ shows human-readable names only; internal domains, IDs, camera layers, and ordering
1889
+ values belong in deliberate debug tooling rather than the primary object list.
1890
+ Pass a `HierarchyPanelCopyConfig` value as `copyConfig` to show one bulk
1891
+ **Copy Config** action. The panel writes that value as formatted JSON and confirms
1892
+ with **Copied Config**. For a complete scene
1893
+ handoff, include `renderGroupOrder`, `renderGroups`, `objects`, and `components`;
1894
+ omit transient editor state such as current selection and panel position. The copied
1895
+ group order therefore contains only groups backed by current scene objects.
1896
+
1897
+ Use this versioned shape for a complete handoff:
1898
+
1899
+ ```ts
1900
+ type SceneConfigV1 = {
1901
+ kind: "scene";
1902
+ version: 1;
1903
+ renderGroupOrder: RenderGroupId[];
1904
+ renderGroups: Record<RenderGroupId, RenderGroup>;
1905
+ objects: Record<SceneObjectId, SceneObject>;
1906
+ components: Record<SceneComponentId, SceneComponent>;
1907
+ };
1908
+ ```
1909
+
1910
+ This is currently an export/debugging contract, not a guaranteed round-trip
1911
+ format: there is no public scene-config importer. Exclude selection, panel
1912
+ position, fold/collapse state, active drag state, feedback messages, and any
1913
+ flat external `VideoBackground`.
1914
+
1915
+ Current compatibility and limits:
1916
+
1917
+ | Domain | Required composition | Status |
1918
+ | ------------------ | -------------------------------------------------- | --------- |
1919
+ | General / 3D Scene | `Transform` + `MeshRenderer` | supported |
1920
+ | 2D Foreground | `ScreenTransform` + `ScreenText` or `ScreenImage` | supported |
1921
+ | Background | editor camera-texture plane or external background | partial |
1922
+ | AR Tracking | tracking scene-object components | reserved |
1923
+ | Post Effects | post-processing scene object | reserved |
1924
+
1925
+ Reserved groups are not materialized or displayed until compatible managed scene
1926
+ objects exist. Each active group adds one scene render, with a maximum of
1927
+ `MAX_RENDER_GROUP_PASSES` (currently 30) managed passes. Import that constant from
1928
+ `@vincentt-xr/sdk` when guarding group creation. `createRenderPlan` omits groups
1929
+ beyond the limit instead of throwing or assigning colliding camera layers.
1930
+ `EXTERNAL_BACKGROUND_RENDER_LAYER` remains temporary infrastructure for legacy
1931
+ external background composition. The reusable scene editor should prefer its
1932
+ own camera-texture plane instead of treating `VideoBackground` as a managed
1933
+ scene object.
1934
+
1935
+ An explicit `renderGroupId` assignment is retained only when its policy accepts the
1936
+ object's components; otherwise grouping uses the inferred compatible domain. Object
1937
+ IDs, component IDs, and component settings remain unchanged.
1938
+
1939
+ The render-group tests protect derived group order, compatible defaults, local
1940
+ object reorder, rejected move no-ops, stable render bands/layers, sequential depth
1941
+ clearing, grouped capture, and interaction across active camera layers. The
1942
+ compiling reference implementation is `/layer-panel-test` in the SDK example.
1943
+
548
1944
  ### Shared debug authoring surface for `ScreenText`
549
1945
 
550
1946
  The SDK also ships a dev-only inspector at `lib/debug-ui/screen-text-settings`
@@ -558,12 +1954,12 @@ including `name`, `kind: "screenText"`, `transform`, `content`, `style`,
558
1954
  `canvasSize`. Use that payload as the handoff from human layout tuning back to
559
1955
  code.
560
1956
 
561
- ### Shared debug layer panel
1957
+ ### Legacy debug layer panel
562
1958
 
563
1959
  The SDK ships a reusable `LayerPanel` in `lib/debug-ui/layer-panel` for
564
- Photoshop-style stack management. It is intended for dev / creator tooling
565
- only and is currently used by the template preview to manage `ScreenImage` and
566
- `ScreenText` layers. Production shipping surfaces should not include the panel.
1960
+ flat-stack compatibility. New grouped editors should use `HierarchyPanel`.
1961
+ Both are intended for dev / creator tooling only. Production shipping surfaces
1962
+ should not include either panel.
567
1963
 
568
1964
  ---
569
1965
 
@@ -663,21 +2059,28 @@ import { AspectRatioContainer } from "@vincentt-xr/sdk";
663
2059
 
664
2060
  Grab a photo of the live render or record the canvas, then save or share it.
665
2061
 
666
- ### `session.captureFrame()` + `dataURLtoFile()` — Single-shot photo
2062
+ ### `useFrameCapture()` + `dataURLtoFile()` — Single-shot photo
667
2063
 
668
- `captureFrame()` forces a fresh render and returns a `data:` URL of the current
669
- frame. `dataURLtoFile(dataUrl, filename)` turns it into a `File` for upload or the
670
- Web Share API.
2064
+ Use `useFrameCapture` for UI/effect logic. It exposes `status`, `capturePhoto`,
2065
+ `clear`, `dataUrl`, and `error`, plus `onCaptureStart`, `onCaptureComplete`, and
2066
+ `onError`. Status is `idle | capturing | completed | error`. The underlying
2067
+ capture forces a fresh render and returns a `data:` URL; `dataURLtoFile` turns
2068
+ it into a `File` for upload or the Web Share API. `capturePhoto()` resolves to
2069
+ the URL on success or `null` after setting `status: "error"` and `error` on
2070
+ failure; callers do not need a rejection handler.
671
2071
 
672
2072
  ```tsx
673
- import { dataURLtoFile } from "@vincentt-xr/sdk";
674
- import { useXRContext } from "@vincentt-xr/sdk/low-level";
2073
+ import { dataURLtoFile, useFrameCapture } from "@vincentt-xr/sdk";
675
2074
 
676
- const { session } = useXRContext();
677
- const dataUrl = session.captureFrame();
678
- const file = dataURLtoFile(dataUrl, "photo.png");
2075
+ const photo = useFrameCapture({
2076
+ onCaptureComplete: (dataUrl) => share(dataURLtoFile(dataUrl, "photo.png")),
2077
+ });
2078
+ await photo.capturePhoto();
679
2079
  ```
680
2080
 
2081
+ For synchronous low-level code, `session.captureFrame()` remains available via
2082
+ `useXRContext` from `@vincentt-xr/sdk/low-level`.
2083
+
681
2084
  ### `useMediaRecorder()` — Record the canvas to a video Blob
682
2085
 
683
2086
  Records a `MediaStream` (from the canvas's `captureStream()`) to a `Blob`. Picks
@@ -694,6 +2097,20 @@ await rec.start(canvas.captureStream(30));
694
2097
  rec.stop(); // -> rec.recordedBlob is the video Blob
695
2098
  ```
696
2099
 
2100
+ Use `rec.status` as the source of truth:
2101
+ `idle | starting | recording | paused | stopping | completed | error`.
2102
+ Controls are `start`, `pause`, `resume`, `stop`, and `clear`. Events are
2103
+ `onRecordingStart`, `onRecordingPause`, `onRecordingResume`,
2104
+ `onRecordingStop`, `onRecordingComplete`, and `onError`. Keep `isRecording`
2105
+ and `isPaused` only as convenience booleans; do not invent parallel state.
2106
+ When `passThroughStream` is configured, call `start()` without an argument;
2107
+ otherwise call `start(stream)` with the canvas capture stream.
2108
+ The verified `/capture-recording` example exercises the complete photo and video
2109
+ lifecycle against the live XR canvas. It displays a recording HUD and elapsed
2110
+ time for `starting`, `recording`, `paused`, `stopping`, and `completed`, then
2111
+ previews the resulting `recordedBlob` in a video player. Photo and recording
2112
+ flows were verified with a real webcam on 26 July 2026.
2113
+
697
2114
  ---
698
2115
 
699
2116
  ## Audio
@@ -753,3 +2170,154 @@ function SoundToggle() {
753
2170
  Audio assets: m4a (AAC). A rare app needing a second, independent mixer can
754
2171
  `new AudioEngine()` and drive it directly; the free functions always target the
755
2172
  one default engine.
2173
+
2174
+ ## 3D behavior components: LookAt and PinToMesh
2175
+
2176
+ `LookAt` and `PinToMesh` are partial-stage XRScene behavior components. They are
2177
+ exported for editor/API prototyping, but they are not yet a fully verified
2178
+ Effect House parity surface.
2179
+
2180
+ `LookAt` rotates its child toward a target object resolved either from a direct
2181
+ `Object3D` prop or from serialized scene-object target ids. Treat it as suitable
2182
+ for current editor experiments, not final production behavior, until point vs
2183
+ direction mode, axis conventions, offset ordering, and missing-target behavior
2184
+ are covered by tests.
2185
+
2186
+ When `SceneObjectRenderer` renders nested objects, child scene objects do not
2187
+ inherit the parent's LookAt reorientation by default. `LookAtSettings` includes
2188
+ `inheritChildren: boolean` for the special case where children should rigidly
2189
+ follow the parent's look-at facing. Defaults from
2190
+ `createDefaultComponent("LookAt", objectId)` set `inheritChildren: false`.
2191
+
2192
+ `PinToMesh` wraps a source object and pins that source wrapper to a target mesh
2193
+ surface. Pass `PinToMeshSettings` with
2194
+ `enabled: true` and a non-null `targetObjectId`; default settings are disabled
2195
+ with no target. The component supports static mesh surface lookup, UV input,
2196
+ orientation, vertex-normal option, offset space, offset position, and offset
2197
+ rotation. Offset position and offset rotation default to `{ x: 0, y: 0, z: 0 }`.
2198
+ When PinToMesh is disabled or the target becomes unavailable, the runtime pin
2199
+ wrapper resets to local position `0,0,0` and identity rotation so the child
2200
+ `Transform3D` remains the authored source of truth.
2201
+
2202
+ Pin wrapper rotation is identity-based for `orientation: "none"` and
2203
+ `orientation: "position"`. `orientation: "positionAndDirection"` aligns the
2204
+ wrapper to the target surface normal before offset rotation is applied. Offset
2205
+ rotation is reapplied from a clean base each frame and must not compound over
2206
+ time.
2207
+
2208
+ Use `PinToMeshSettingsPanel` from `@vincentt-xr/sdk/debug-ui` for editor
2209
+ controls. The panel uses Transform-style text number inputs for UV and offsets:
2210
+ visible zero defaults, decimal validation, 0.1 arrow-step increments, Shift for
2211
+ larger steps, and Alt for finer steps. Serialized scenes should store stable
2212
+ scene-object ids for targets; runtime renderers resolve those ids to registered
2213
+ `Object3D` instances.
2214
+
2215
+ Direct runtime usage:
2216
+
2217
+ ```tsx
2218
+ import { LookAt, PinToMesh, Transform3D, MeshRenderer } from "@vincentt-xr/sdk";
2219
+
2220
+ <PinToMesh
2221
+ target={targetObject}
2222
+ settings={{
2223
+ enabled: true,
2224
+ targetObjectId: "base-plane",
2225
+ orientation: "positionAndDirection",
2226
+ uv: { x: 0.5, y: 0.5 },
2227
+ vertexNormal: true,
2228
+ offsetSpace: "local",
2229
+ offsetPosition: { x: 0, y: 0.25, z: 0 },
2230
+ offsetRotation: { x: 0, y: 0, z: 0 },
2231
+ }}
2232
+ >
2233
+ <Transform3D value={transformSettings}>
2234
+ <LookAt
2235
+ target={lookTargetObject}
2236
+ settings={{
2237
+ enabled: true,
2238
+ targetObjectId: "target-object",
2239
+ mode: "point",
2240
+ offsetRotation: { x: 0, y: 0, z: 0 },
2241
+ }}
2242
+ >
2243
+ <MeshRenderer value={meshSettings} />
2244
+ </LookAt>
2245
+ </Transform3D>
2246
+ </PinToMesh>;
2247
+ ```
2248
+
2249
+ Serialized component defaults:
2250
+
2251
+ - `createDefaultComponent("LookAt", objectId)` returns disabled LookAt settings
2252
+ with `targetObjectId: null`, `mode: "point"`, and zero offset rotation.
2253
+ - `createDefaultComponent("PinToMesh", objectId)` returns disabled PinToMesh
2254
+ settings with `targetObjectId: null`, UV `{ x: 0.5, y: 0.5 }`,
2255
+ `orientation: "none"`, `vertexNormal: false`, `offsetSpace: "local"`, and
2256
+ zero position/rotation offsets.
2257
+
2258
+ Current limitations:
2259
+
2260
+ - UV behavior is not final. Missing UVs currently fall back to target bounds
2261
+ center behavior instead of a visible invalid-target state.
2262
+ - LookAt has a minimal API: `enabled`, `targetObjectId`, `mode`, and
2263
+ `offsetRotation`. Axis remapping and world-up controls are still planned.
2264
+ - Runtime target resolution is scoped by `XRProvider` through an internal
2265
+ runtime-object registry. Direct calls to `registerRuntimeObject` and
2266
+ `resolveRuntimeObject` outside an `XRProvider` use a legacy fallback registry
2267
+ and should be avoided for multi-scene pages.
2268
+
2269
+ ## 3D light scene objects
2270
+
2271
+ `LightObject` renders a serialized light component through `Transform3D`.
2272
+ Import from `@vincentt-xr/sdk`.
2273
+
2274
+ ```tsx
2275
+ import { LightObject } from "@vincentt-xr/sdk";
2276
+
2277
+ <LightObject
2278
+ transform={transformSettings}
2279
+ settings={{
2280
+ enabled: true,
2281
+ type: "directional",
2282
+ color: "#ffffff",
2283
+ intensity: 2.2,
2284
+ castShadow: true,
2285
+ distance: 10,
2286
+ angle: Math.PI / 6,
2287
+ penumbra: 0.2,
2288
+ decay: 2,
2289
+ showHelper: false,
2290
+ }}
2291
+ />;
2292
+ ```
2293
+
2294
+ `LightSettings.type` is `"ambient" | "directional" | "point" | "spot"`.
2295
+ Directional, point, and spot lights support `castShadow`; ambient lights only
2296
+ contribute global light. `showHelper` draws a small non-physical editor marker
2297
+ for the light object and defaults to `false`; enable it only in editor/debug
2298
+ surfaces.
2299
+
2300
+ For an Effect House-style declarative player, use `<AudioPlayer>`. It supports
2301
+ `once`, a finite `loop` count, and `infinite` playback, plus autoplay intent,
2302
+ per-player mute, and a clamped 0–100 volume. Browser autoplay policy still
2303
+ applies: audible autoplay may reject until a user gesture, surfaced through
2304
+ `onError`. The component is headless (it renders a hidden `<audio>`); attach a
2305
+ ref for custom controls.
2306
+
2307
+ ```tsx
2308
+ import { AudioPlayer, type AudioPlayerHandle } from "@vincentt-xr/sdk";
2309
+
2310
+ const player = useRef<AudioPlayerHandle>(null);
2311
+
2312
+ <AudioPlayer
2313
+ ref={player}
2314
+ src={musicUrl}
2315
+ playMode="loop"
2316
+ loopCount={3}
2317
+ autoplay
2318
+ volume={80}
2319
+ onError={(error) => console.warn(error)}
2320
+ />;
2321
+
2322
+ // From a user gesture: player.current?.play(), .pause(), or .stop().
2323
+ ```