@reactvision/react-viro 2.55.0 → 2.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +27 -0
  2. package/android/react_viro/react_viro-release.aar +0 -0
  3. package/android/viro_renderer/viro_renderer-release.aar +0 -0
  4. package/components/AR/ViroARScene.tsx +17 -0
  5. package/components/AR/ViroARSceneNavigator.tsx +33 -0
  6. package/components/Utilities/StreamingAudioManager.ts +73 -0
  7. package/components/Utilities/ViroVersion.ts +1 -1
  8. package/components/ViroCameraTexture.tsx +230 -0
  9. package/components/ViroGameLoop.tsx +74 -0
  10. package/components/ViroGameLoopUtils.ts +52 -0
  11. package/components/ViroObjectDetector.tsx +222 -0
  12. package/components/ViroVirtualButton.tsx +101 -0
  13. package/components/ViroVirtualJoystick.tsx +126 -0
  14. package/components/hooks/useGameLoop.ts +78 -0
  15. package/dist/components/AR/ViroARScene.d.ts +12 -0
  16. package/dist/components/AR/ViroARScene.js +8 -1
  17. package/dist/components/AR/ViroARSceneNavigator.d.ts +30 -0
  18. package/dist/components/Utilities/StreamingAudioManager.d.ts +22 -0
  19. package/dist/components/Utilities/StreamingAudioManager.js +45 -0
  20. package/dist/components/Utilities/ViroVersion.d.ts +1 -1
  21. package/dist/components/Utilities/ViroVersion.js +1 -1
  22. package/dist/components/ViroCameraTexture.d.ts +137 -0
  23. package/dist/components/ViroCameraTexture.js +170 -0
  24. package/dist/components/ViroGameLoop.d.ts +31 -0
  25. package/dist/components/ViroGameLoop.js +31 -0
  26. package/dist/components/ViroGameLoopUtils.d.ts +26 -0
  27. package/dist/components/ViroGameLoopUtils.js +44 -0
  28. package/dist/components/ViroObjectDetector.d.ts +138 -0
  29. package/dist/components/ViroObjectDetector.js +85 -0
  30. package/dist/components/ViroVirtualButton.d.ts +76 -0
  31. package/dist/components/ViroVirtualButton.js +62 -0
  32. package/dist/components/ViroVirtualJoystick.d.ts +85 -0
  33. package/dist/components/ViroVirtualJoystick.js +72 -0
  34. package/dist/components/hooks/useGameLoop.d.ts +34 -0
  35. package/dist/components/hooks/useGameLoop.js +63 -0
  36. package/dist/index.d.ts +11 -1
  37. package/dist/index.js +20 -2
  38. package/dist/plugins/withViro.js +2 -15
  39. package/dist/plugins/withViroAndroid.js +52 -15
  40. package/dist/plugins/withViroIos.js +0 -6
  41. package/index.ts +21 -0
  42. package/ios/ViroReact.podspec +25 -6
  43. package/ios/dist/ViroRenderer/ViroKit.framework/ARCoreCoreMLSemanticsResources.bundle/Info.plist +0 -0
  44. package/ios/dist/ViroRenderer/ViroKit.framework/ARCoreResources.bundle/Info.plist +0 -0
  45. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARDepthMesh.h +9 -1
  46. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrame.h +33 -0
  47. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrameiOS.h +26 -0
  48. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARScene.h +15 -1
  49. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSceneDelegateiOS.h +8 -0
  50. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSession.h +7 -0
  51. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSessioniOS.h +18 -0
  52. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARWorldMesh.h +111 -8
  53. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROAudioPlayer.h +19 -0
  54. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROAudioPlayerStreamiOS.h +95 -0
  55. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROCameraTextureiOS.h +17 -1
  56. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODriver.h +20 -0
  57. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODynamicGeometry.h +170 -0
  58. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODynamicMeshNode.h +87 -0
  59. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROGLTFLoader.h +5 -1
  60. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROGameLoopListener.h +125 -0
  61. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROInputState.h +133 -0
  62. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROMonocularDepthEstimator.h +73 -0
  63. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROObjectDetector.h +134 -0
  64. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROPCMRingBuffer.h +143 -0
  65. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROPencil.h +9 -0
  66. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRORenderContext.h +4 -0
  67. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRORenderer.h +1 -0
  68. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROVertexBuffer.h +102 -0
  69. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROViewAR.h +9 -0
  70. package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROVirtualControllerRegistry.h +106 -0
  71. package/ios/dist/ViroRenderer/ViroKit.framework/Info.plist +0 -0
  72. package/ios/dist/ViroRenderer/ViroKit.framework/Shaders.dat +1 -1
  73. package/ios/dist/ViroRenderer/ViroKit.framework/ViroKit +0 -0
  74. package/ios/dist/ViroRenderer/ViroKit.framework/constant_fsh.glsl +1 -0
  75. package/ios/dist/ViroRenderer/ViroKit.framework/constant_fsh_adreno.glsl +1 -0
  76. package/ios/dist/ViroRenderer/ViroKit.framework/standard_fsh.glsl +1 -0
  77. package/ios/dist/ViroRenderer/ViroKit.framework/standard_vsh.glsl +4 -0
  78. package/ios/dist/ViroRenderer/ViroKit.framework/standard_vsh_adreno.glsl +4 -0
  79. package/ios/dist/ViroRendererVisionOS/ViroKit.podspec +27 -0
  80. package/ios/dist/include/VRTARScene.h +1 -0
  81. package/ios/dist/include/VRTARSceneNavigator.h +3 -0
  82. package/ios/dist/include/VRTCameraTexture.h +62 -0
  83. package/ios/dist/include/VRTCameraTextureManager.h +30 -0
  84. package/ios/dist/include/VRTCameraTextureModule.h +38 -0
  85. package/ios/dist/include/VRTGameLoopView.h +19 -0
  86. package/ios/dist/include/VRTGameLoopViewManager.h +8 -0
  87. package/ios/dist/include/VRTManagedAnimation.h +67 -0
  88. package/ios/dist/include/VRTStreamingAudioModule.h +21 -0
  89. package/ios/dist/include/VRTVirtualButtonView.h +47 -0
  90. package/ios/dist/include/VRTVirtualButtonViewManager.h +21 -0
  91. package/ios/dist/include/VRTVirtualJoystickView.h +59 -0
  92. package/ios/dist/include/VRTVirtualJoystickViewManager.h +26 -0
  93. package/ios/dist/lib/libViroReact.a +0 -0
  94. package/package.json +17 -20
@@ -0,0 +1,125 @@
1
+ //
2
+ // VROGameLoopListener.h
3
+ // ViroKit
4
+ //
5
+ // Copyright © 2026 ReactVision. All rights reserved.
6
+ //
7
+ // Permission is hereby granted, free of charge, to any person obtaining
8
+ // a copy of this software and associated documentation files (the
9
+ // "Software"), to deal in the Software without restriction, including
10
+ // without limitation the rights to use, copy, modify, merge, publish,
11
+ // distribute, sublicense, and/or sell copies of the Software, and to
12
+ // permit persons to whom the Software is furnished to do so, subject to
13
+ // the following conditions:
14
+ //
15
+ // The above copyright notice and this permission notice shall be included
16
+ // in all copies or substantial portions of the Software.
17
+
18
+ #ifndef VROGameLoopListener_h
19
+ #define VROGameLoopListener_h
20
+
21
+ #include <functional>
22
+ #include "VROFrameListener.h"
23
+ #include "VROTime.h"
24
+
25
+ /*
26
+ VROGameLoopListener — thin VROFrameListener wrapper that tracks delta time and
27
+ elapsed time, then fires typed callbacks from the render thread.
28
+
29
+ Typical use (C++, from native bridge code):
30
+
31
+ auto loop = std::make_shared<VROGameLoopListener>();
32
+ loop->setOnFrameWillRender([](float dt, float elapsed) {
33
+ // game logic here — runs before physics and rendering
34
+ });
35
+ frameSynchronizer->addFrameListener(loop);
36
+
37
+ For fixed-step simulation (F5 addendum):
38
+
39
+ loop->setFixedHz(30);
40
+ loop->setOnFixedStep([](float dt) {
41
+ // guaranteed 1/30 s ticks
42
+ });
43
+
44
+ Thread safety: callbacks are invoked on the render thread. Callers are
45
+ responsible for marshalling to another thread if needed.
46
+ */
47
+ class VROGameLoopListener : public VROFrameListener {
48
+ public:
49
+ using UpdateCallback = std::function<void(float dtSec, float elapsedSec)>;
50
+ using FixedStepCallback = std::function<void(float dtSec)>;
51
+
52
+ VROGameLoopListener() = default;
53
+ ~VROGameLoopListener() override = default;
54
+
55
+ // ── Variable-step callbacks ────────────────────────────────────────────────
56
+
57
+ void setOnFrameWillRender(UpdateCallback cb) { _onWillRender = std::move(cb); }
58
+ void setOnFrameDidRender(UpdateCallback cb) { _onDidRender = std::move(cb); }
59
+
60
+ // ── Fixed-step (F5 addendum) ───────────────────────────────────────────────
61
+
62
+ /*
63
+ Set the fixed simulation frequency in Hz (e.g. 30 for libsm64, 60 for typical
64
+ physics). Each onFrameWillRender call drains the accumulator, firing the fixed-
65
+ step callback as many times as needed to stay in sync with wall time.
66
+ A clamped dt prevents a "spiral of death" when frames are very slow.
67
+ */
68
+ void setFixedHz(float hz) {
69
+ _fixedHz = hz;
70
+ _fixedDt = (hz > 0.f) ? 1.f / hz : 0.f;
71
+ _accumulator = 0.f;
72
+ _fixedEnabled = (hz > 0.f);
73
+ }
74
+ void setOnFixedStep(FixedStepCallback cb) { _onFixedStep = std::move(cb); }
75
+
76
+ // ── VROFrameListener ──────────────────────────────────────────────────────
77
+
78
+ void onFrameWillRender(const VRORenderContext &context) override {
79
+ double now = VROTimeCurrentMillis();
80
+ if (_startTime < 0.0) _startTime = _lastTime = now;
81
+
82
+ float dt = (float)((now - _lastTime) / 1000.0);
83
+ float elapsed = (float)((now - _startTime) / 1000.0);
84
+ _lastTime = now;
85
+
86
+ // Fixed-step drain — clamp frame dt to avoid spiral of death
87
+ if (_fixedEnabled && _onFixedStep && _fixedDt > 0.f) {
88
+ float safeDt = (dt < kMaxFrameDt) ? dt : kMaxFrameDt;
89
+ _accumulator += safeDt;
90
+ while (_accumulator >= _fixedDt) {
91
+ _onFixedStep(_fixedDt);
92
+ _accumulator -= _fixedDt;
93
+ }
94
+ }
95
+
96
+ if (_onWillRender) _onWillRender(dt, elapsed);
97
+ }
98
+
99
+ void onFrameDidRender(const VRORenderContext &context) override {
100
+ if (!_onDidRender) return;
101
+ double now = VROTimeCurrentMillis();
102
+ float dt = (float)((now - _lastTime) / 1000.0);
103
+ float elapsed = (float)((now - _startTime) / 1000.0);
104
+ _onDidRender(dt, elapsed);
105
+ }
106
+
107
+ private:
108
+ // Maximum frame delta clamped in fixed-step mode (prevents spiral of death
109
+ // if the app is paused or debugging halts the render loop).
110
+ static constexpr float kMaxFrameDt = 0.25f; // 4 fps floor
111
+
112
+ UpdateCallback _onWillRender;
113
+ UpdateCallback _onDidRender;
114
+ FixedStepCallback _onFixedStep;
115
+
116
+ double _startTime = -1.0;
117
+ double _lastTime = -1.0;
118
+
119
+ bool _fixedEnabled = false;
120
+ float _fixedHz = 0.f;
121
+ float _fixedDt = 0.f;
122
+ float _accumulator = 0.f;
123
+ };
124
+
125
+ #endif /* VROGameLoopListener_h */
@@ -0,0 +1,133 @@
1
+ //
2
+ // VROInputState.h
3
+ // ViroKit
4
+ //
5
+ // Copyright © 2026 ReactVision. All rights reserved.
6
+ //
7
+ // Permission is hereby granted, free of charge, to any person obtaining
8
+ // a copy of this software and associated documentation files (the
9
+ // "Software"), to deal in the Software without restriction, including
10
+ // without limitation the rights to use, copy, modify, merge, publish,
11
+ // distribute, sublicense, and/or sell copies of the Software, and to
12
+ // permit persons to whom the Software is furnished to do so, subject to
13
+ // the following conditions:
14
+ //
15
+ // The above copyright notice and this permission notice shall be included
16
+ // in all copies or substantial portions of the Software.
17
+
18
+ #ifndef VROInputState_h
19
+ #define VROInputState_h
20
+
21
+ #include <stdint.h>
22
+ #include <mutex>
23
+
24
+ /*
25
+ Unified controller state for game-style continuous input. Filled by zero or more
26
+ source adapters (virtual touch joystick, MFi/Bluetooth gamepad, device tilt,
27
+ voice, hand-tracking) and read by consumers (game loop, simulation adapter,
28
+ native modules forwarding to JS).
29
+
30
+ Independent of VROInputControllerBase / VROInputPresenter, which are oriented at
31
+ pointer/click events for raycasting in AR/VR scenes. VROInputState models the
32
+ "continuous controller snapshot" use case: per-frame stick deflections, button
33
+ hold states, analog triggers.
34
+
35
+ Thread-safety: write methods are safe to call from any thread (touch handler,
36
+ gamepad callback, sensor thread). Readers call snapshot() to get a consistent
37
+ view of the entire state in one shot. Internal mutex contention is negligible
38
+ because each writer touches the state at most a few times per frame.
39
+
40
+ Button indices are caller-defined; the table below documents the recommended
41
+ default mapping for game-style apps. Adapters can map their physical inputs
42
+ to any index they like.
43
+ */
44
+
45
+ class VROInputState {
46
+
47
+ public:
48
+
49
+ static constexpr int kMaxButtons = 32;
50
+
51
+ /*
52
+ Recommended button-index mapping for SM64-style inputs and most casual
53
+ games. Adapters that target this layout populate buttons at these indices;
54
+ consumers that read this layout know what each bit means. Mappings are
55
+ not enforced — they are conventions.
56
+ */
57
+ enum DefaultButton : int {
58
+ Button_A = 0, // jump / primary action
59
+ Button_B = 1, // attack / grab / secondary action
60
+ Button_X = 2,
61
+ Button_Y = 3,
62
+ Button_Z = 4, // crouch / ground-pound
63
+ Button_L1 = 5,
64
+ Button_R1 = 6,
65
+ Button_L2 = 7,
66
+ Button_R2 = 8,
67
+ Button_Start = 9,
68
+ Button_Select = 10,
69
+ Button_StickLClick = 11,
70
+ Button_StickRClick = 12,
71
+ Button_DPad_Up = 13,
72
+ Button_DPad_Down = 14,
73
+ Button_DPad_Left = 15,
74
+ Button_DPad_Right = 16,
75
+ // 17..31 free for app-specific buttons
76
+ };
77
+
78
+ /*
79
+ Plain-data snapshot returned by snapshot() and consumed by readers. Safe
80
+ to copy and pass across threads.
81
+ */
82
+ struct Snapshot {
83
+ float stickLX = 0.f;
84
+ float stickLY = 0.f;
85
+ float stickRX = 0.f;
86
+ float stickRY = 0.f;
87
+ float triggerL = 0.f;
88
+ float triggerR = 0.f;
89
+ uint32_t buttonBits = 0; // bit i = button i state (1 = pressed)
90
+
91
+ bool getButton(int idx) const {
92
+ if (idx < 0 || idx >= kMaxButtons) return false;
93
+ return (buttonBits & (1u << idx)) != 0;
94
+ }
95
+ };
96
+
97
+ VROInputState() = default;
98
+
99
+ /*
100
+ Reader API. Returns a consistent snapshot of every field. Cheap; takes the
101
+ mutex once.
102
+ */
103
+ Snapshot snapshot() const;
104
+
105
+ /*
106
+ Writer API. Each call takes the mutex briefly. Adapters can call these
107
+ from any thread.
108
+
109
+ Stick axis convention: x in [-1, 1] left-to-right, y in [-1, 1]
110
+ down-to-up. Adapters are responsible for sign conventions and clamping;
111
+ values outside [-1, 1] are accepted but consumers may behave oddly.
112
+ */
113
+ void setStickL(float x, float y);
114
+ void setStickR(float x, float y);
115
+ void setTriggerL(float value);
116
+ void setTriggerR(float value);
117
+ void setButton(int idx, bool pressed);
118
+
119
+ /*
120
+ Reset all fields to neutral (sticks zeroed, triggers zeroed, no buttons
121
+ pressed). Called when a source adapter unregisters or when a controller
122
+ is intentionally cleared (e.g. on app background).
123
+ */
124
+ void reset();
125
+
126
+ private:
127
+
128
+ mutable std::mutex _mtx;
129
+ Snapshot _state;
130
+
131
+ };
132
+
133
+ #endif /* VROInputState_h */
@@ -82,6 +82,15 @@ public:
82
82
  * estimator->update(arFrame);
83
83
  * auto depthTexture = estimator->getDepthTexture();
84
84
  */
85
+ /**
86
+ * Controls how the monocular depth estimator derives metric scale.
87
+ */
88
+ enum class VROMonocularDepthCalibration {
89
+ None, // no scaling (raw model output, scale = 1.0)
90
+ Manual, // caller sets scale via setScaleFactor() (default)
91
+ LiDARReference, // stub: sample LiDAR to auto-derive scale (falls back to Manual)
92
+ };
93
+
85
94
  class API_AVAILABLE(ios(14.0)) VROMonocularDepthEstimator :
86
95
  public std::enable_shared_from_this<VROMonocularDepthEstimator> {
87
96
 
@@ -126,6 +135,16 @@ public:
126
135
  */
127
136
  void update(const VROARFrame *frame);
128
137
 
138
+ /**
139
+ * Run a single throwaway inference on a blank image to force CoreML / Neural Engine
140
+ * model specialization up front. Without this, the FIRST real inference pays the
141
+ * full ANE compile cost (many seconds on A-series chips) — the dominant cause of the
142
+ * long delay before the first depth frame appears. Safe to call once right after
143
+ * initWithModel(); runs asynchronously on the depth queue and does not touch the
144
+ * live depth buffers (it uses a private throwaway request).
145
+ */
146
+ void warmup();
147
+
129
148
  #pragma mark - Depth Output
130
149
 
131
150
  /**
@@ -161,11 +180,28 @@ public:
161
180
  * Set the scale factor for converting model output to metric depth.
162
181
  * Some models output relative/disparity depth that needs scaling.
163
182
  * Default is 1.0 (assumes model outputs metric depth in meters).
183
+ * Implicitly sets calibration mode to Manual.
164
184
  *
165
185
  * @param scale The scale factor to multiply depth values by.
166
186
  */
167
187
  void setScaleFactor(float scale);
168
188
 
189
+ /**
190
+ * Set the calibration mode that controls how depth scale is derived.
191
+ * - None: raw model output (scale = 1.0)
192
+ * - Manual: caller-supplied scale via setScaleFactor() (default)
193
+ * - LiDARReference: stub, logs warning and falls back to Manual
194
+ */
195
+ void setCalibrationMode(VROMonocularDepthCalibration mode);
196
+ VROMonocularDepthCalibration getCalibrationMode() const;
197
+
198
+ /**
199
+ * Flush any pending depth data to the GPU texture.
200
+ * Must be called on the render (GL) thread. VROARSessioniOS calls this
201
+ * automatically each frame before depth is consumed.
202
+ */
203
+ void flushPendingDepthUpdate();
204
+
169
205
  /**
170
206
  * Enable or disable temporal filtering for depth stability.
171
207
  * When enabled, depth values are smoothed across frames using
@@ -232,6 +268,36 @@ public:
232
268
  */
233
269
  int getDepthBufferHeight() const { return _depthHeight; }
234
270
 
271
+ /**
272
+ * Atomically snapshot the depth and confidence buffers for offline processing
273
+ * (e.g. world-mesh generation). Thread-safe — takes a copy under the depth mutex.
274
+ * Returns false if no data is available.
275
+ */
276
+ bool snapshotDepthBuffers(std::vector<float> &outDepth,
277
+ std::vector<float> &outConfidence,
278
+ int &outWidth, int &outHeight) const;
279
+
280
+ /**
281
+ * Sample the synthesized per-pixel confidence at depth-texture UV coordinates.
282
+ * Returns a value in [0, 1]: 1 = highly stable, 0 = discontinuity or invalid.
283
+ * Returns -1 if no confidence data is available (temporal filtering disabled or
284
+ * first frame). Thread-safe.
285
+ *
286
+ * @param u Depth texture U coordinate [0, 1].
287
+ * @param v Depth texture V coordinate [0, 1].
288
+ */
289
+ float sampleConfidenceAtDepthUV(float u, float v) const;
290
+
291
+ /**
292
+ * Set the confidence threshold for hit-test upgrade decisions on the monocular path.
293
+ * Hit results are upgraded to DepthPoint only when synthesized confidence exceeds
294
+ * this value. Default is 0.3.
295
+ *
296
+ * @param threshold Confidence threshold in [0, 1].
297
+ */
298
+ void setHitTestConfidenceThreshold(float threshold);
299
+ float getHitTestConfidenceThreshold() const;
300
+
235
301
  private:
236
302
  // Graphics driver
237
303
  std::weak_ptr<VRODriver> _driver;
@@ -258,17 +324,24 @@ private:
258
324
  std::shared_ptr<VROTexture> _currentDepthTexture;
259
325
  std::vector<float> _depthBuffer;
260
326
  std::vector<float> _previousDepthBuffer;
327
+ std::vector<float> _confidenceBuffer; // per-pixel synthesized confidence [0,1]
261
328
  int _depthWidth;
262
329
  int _depthHeight;
263
330
  VROMatrix4f _depthTextureTransform;
264
331
 
265
332
  // Configuration
266
333
  float _depthScaleFactor;
334
+ VROMonocularDepthCalibration _calibrationMode;
335
+ float _hitTestConfidenceThreshold;
267
336
  bool _temporalFilteringEnabled;
268
337
  float _temporalFilterAlpha;
269
338
  int _targetFPS;
270
339
  double _lastInferenceTime;
271
340
 
341
+ // W4: staging buffer for in-place GPU texture updates
342
+ std::vector<float> _stagingDepthBuffer;
343
+ std::atomic<bool> _stagingDirty;
344
+
272
345
  // Diagnostics
273
346
  float _currentFPS;
274
347
  float _averageLatencyMs;
@@ -0,0 +1,134 @@
1
+ //
2
+ // VROObjectDetector.h
3
+ // ViroRenderer
4
+ //
5
+ // Copyright © 2026 ReactVision. All rights reserved.
6
+ //
7
+ // Permission is hereby granted, free of charge, to any person obtaining
8
+ // a copy of this software and associated documentation files (the
9
+ // "Software"), to deal in the Software without restriction, including
10
+ // without limitation the rights to use, copy, modify, merge, publish,
11
+ // distribute, sublicense, and/or sell copies of the Software, and to
12
+ // permit persons to whom the Software is furnished to do so, subject to
13
+ // the following conditions:
14
+ //
15
+ // The above copyright notice and this permission notice shall be included
16
+ // in all copies or substantial portions of the Software.
17
+ //
18
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21
+ // IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
22
+ // CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23
+ // TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24
+ // SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ #ifndef VROObjectDetector_h
27
+ #define VROObjectDetector_h
28
+
29
+ #include <string>
30
+ #include <vector>
31
+ #include <functional>
32
+ #include <memory>
33
+
34
+ /*
35
+ Bounding box in normalized image space [0, 1].
36
+ Origin is top-left corner of the image.
37
+ */
38
+ struct VRODetectionBoundingBox {
39
+ float x; // left edge
40
+ float y; // top edge
41
+ float width;
42
+ float height;
43
+ };
44
+
45
+ struct VRODetectedObject {
46
+ std::string label;
47
+ float confidence;
48
+ VRODetectionBoundingBox boundingBox;
49
+ };
50
+
51
+ /*
52
+ Inference mode matching YOLOE's three prompting strategies:
53
+ - PromptFree: LRPC head, 4,585 built-in RAM++ categories, no runtime prompt needed.
54
+ - Text: RepRTA, detect arbitrary categories given a list of text labels.
55
+ - Visual: SAVPE, detect objects similar to a reference image crop.
56
+ */
57
+ enum class VRODetectorMode {
58
+ PromptFree,
59
+ Text,
60
+ Visual,
61
+ };
62
+
63
+ using VRODetectionCallback = std::function<void(const std::vector<VRODetectedObject> &)>;
64
+
65
+ /*
66
+ Abstract base class for on-device object detectors.
67
+
68
+ Platform implementations (iOS: CoreML / ONNX Runtime, Android: ONNX Runtime)
69
+ subclass this and implement loadModel() and detectInBuffer(). The bridge layer
70
+ owns the camera pipeline and calls detectInBuffer() on each sampled frame.
71
+ */
72
+ class VROObjectDetector {
73
+ public:
74
+ virtual ~VROObjectDetector() {}
75
+
76
+ /*
77
+ Load the model from the given path.
78
+ On iOS: path to a .mlpackage (CoreML) or .onnx file.
79
+ On Android: path to a .onnx file inside the app's assets or files dir.
80
+ Returns true on success.
81
+ */
82
+ virtual bool loadModel(const std::string &modelPath) = 0;
83
+
84
+ /*
85
+ Run detection on a raw RGB pixel buffer.
86
+ - pixels: pointer to contiguous RGB888 data (row-major)
87
+ - width / height: image dimensions in pixels
88
+ Results are delivered asynchronously via the detection callback.
89
+ */
90
+ virtual void detectInBuffer(const uint8_t *pixels, int width, int height) = 0;
91
+
92
+ // --- configuration (call before loadModel or between frames) ---
93
+
94
+ void setMode(VRODetectorMode mode) {
95
+ _mode = mode;
96
+ }
97
+
98
+ /*
99
+ Text categories for Text mode (e.g. {"chair", "person", "laptop"}).
100
+ Ignored in PromptFree and Visual modes.
101
+ */
102
+ void setCategories(const std::vector<std::string> &categories) {
103
+ _categories = categories;
104
+ }
105
+
106
+ void setConfidenceThreshold(float threshold) {
107
+ _confidenceThreshold = threshold;
108
+ }
109
+
110
+ /*
111
+ NMS IoU threshold used during post-processing.
112
+ */
113
+ void setIouThreshold(float threshold) {
114
+ _iouThreshold = threshold;
115
+ }
116
+
117
+ void setCallback(VRODetectionCallback callback) {
118
+ _callback = callback;
119
+ }
120
+
121
+ VRODetectorMode getMode() const { return _mode; }
122
+ float getConfidenceThreshold() const { return _confidenceThreshold; }
123
+ float getIouThreshold() const { return _iouThreshold; }
124
+ const std::vector<std::string> &getCategories() const { return _categories; }
125
+
126
+ protected:
127
+ VRODetectorMode _mode = VRODetectorMode::PromptFree;
128
+ std::vector<std::string> _categories;
129
+ float _confidenceThreshold = 0.4f;
130
+ float _iouThreshold = 0.45f;
131
+ VRODetectionCallback _callback;
132
+ };
133
+
134
+ #endif /* VROObjectDetector_h */
@@ -0,0 +1,143 @@
1
+ //
2
+ // VROPCMRingBuffer.h
3
+ // ViroRenderer
4
+ //
5
+ // Copyright © 2026 ReactVision. All rights reserved.
6
+ //
7
+ // Permission is hereby granted, free of charge, to any person obtaining
8
+ // a copy of this software and associated documentation files (the
9
+ // "Software"), to deal in the Software without restriction, including
10
+ // without limitation the rights to use, copy, modify, merge, publish,
11
+ // distribute, sublicense, and/or sell copies of the Software, and to
12
+ // permit persons to whom the Software is furnished to do so, subject to
13
+ // the following conditions:
14
+ //
15
+ // The above copyright notice and this permission notice shall be included
16
+ // in all copies or substantial portions of the Software.
17
+
18
+ #ifndef VROPCMRingBuffer_h
19
+ #define VROPCMRingBuffer_h
20
+
21
+ #include <vector>
22
+ #include <atomic>
23
+ #include <algorithm>
24
+ #include <cstring>
25
+
26
+ /*
27
+ Lock-free single-producer / single-consumer ring buffer for interleaved float PCM
28
+ samples. Designed for the streaming audio path:
29
+
30
+ Producer (any thread): calls write() with chunks of samples from the
31
+ simulation / TTS / synthesiser.
32
+
33
+ Consumer (audio callback, real-time thread): calls read() to fill the OS
34
+ audio buffer. Must never block — silence (zeros) is returned when the
35
+ ring is empty.
36
+
37
+ Capacity is set at construction and is a power of two so that the index wrap
38
+ is a bitwise AND rather than a modulo. A capacity of 8192 samples at 32 kHz
39
+ stereo gives ~128 ms of headroom, sufficient to absorb a typical JS bridge
40
+ round-trip.
41
+
42
+ NOT safe for multiple producers or multiple consumers.
43
+ */
44
+ class VROPCMRingBuffer {
45
+
46
+ public:
47
+
48
+ explicit VROPCMRingBuffer(size_t capacitySamples = 8192)
49
+ : _capacity(nextPow2(capacitySamples)),
50
+ _mask(_capacity - 1),
51
+ _buffer(_capacity, 0.0f),
52
+ _writePos(0),
53
+ _readPos(0) {}
54
+
55
+ /*
56
+ Write interleaved PCM samples. Called from the producer thread.
57
+ Returns the number of samples actually written (may be less than count
58
+ if the ring is nearly full — caller should retry or drop).
59
+ */
60
+ size_t write(const float *data, size_t count) {
61
+ size_t w = _writePos.load(std::memory_order_relaxed);
62
+ size_t r = _readPos.load(std::memory_order_acquire);
63
+ size_t available = _capacity - (w - r); // free slots
64
+ size_t toWrite = std::min(count, available);
65
+ if (toWrite == 0) return 0;
66
+
67
+ size_t idx = w & _mask;
68
+ size_t firstPart = std::min(toWrite, _capacity - idx);
69
+ memcpy(_buffer.data() + idx, data, firstPart * sizeof(float));
70
+ if (toWrite > firstPart) {
71
+ memcpy(_buffer.data(), data + firstPart,
72
+ (toWrite - firstPart) * sizeof(float));
73
+ }
74
+ _writePos.store(w + toWrite, std::memory_order_release);
75
+ return toWrite;
76
+ }
77
+
78
+ /*
79
+ Read interleaved PCM samples into out. Called from the audio callback
80
+ (consumer thread). Fills with silence if fewer samples are available
81
+ than requested. Returns number of valid samples read.
82
+ */
83
+ size_t read(float *out, size_t count) {
84
+ size_t r = _readPos.load(std::memory_order_relaxed);
85
+ size_t w = _writePos.load(std::memory_order_acquire);
86
+ size_t available = w - r;
87
+ size_t toRead = std::min(count, available);
88
+
89
+ if (toRead > 0) {
90
+ size_t idx = r & _mask;
91
+ size_t firstPart = std::min(toRead, _capacity - idx);
92
+ memcpy(out, _buffer.data() + idx, firstPart * sizeof(float));
93
+ if (toRead > firstPart) {
94
+ memcpy(out + firstPart, _buffer.data(),
95
+ (toRead - firstPart) * sizeof(float));
96
+ }
97
+ _readPos.store(r + toRead, std::memory_order_release);
98
+ }
99
+
100
+ // Zero-fill any unfulfilled frames (silence on underrun).
101
+ if (toRead < count) {
102
+ memset(out + toRead, 0, (count - toRead) * sizeof(float));
103
+ }
104
+ return toRead;
105
+ }
106
+
107
+ int sampleRate() const { return _sampleRate; }
108
+ int channels() const { return _channels; }
109
+
110
+ void setSampleRate(int sr) { _sampleRate = sr; }
111
+ void setChannels(int ch) { _channels = ch; }
112
+
113
+ size_t capacity() const { return _capacity; }
114
+
115
+ /*
116
+ Approximate number of samples currently buffered. Informational only —
117
+ may be slightly stale by the time the caller acts on it.
118
+ */
119
+ size_t available() const {
120
+ return _writePos.load(std::memory_order_relaxed) -
121
+ _readPos.load(std::memory_order_relaxed);
122
+ }
123
+
124
+ private:
125
+
126
+ static size_t nextPow2(size_t n) {
127
+ size_t p = 1;
128
+ while (p < n) p <<= 1;
129
+ return p;
130
+ }
131
+
132
+ const size_t _capacity;
133
+ const size_t _mask;
134
+ std::vector<float> _buffer;
135
+
136
+ std::atomic<size_t> _writePos;
137
+ std::atomic<size_t> _readPos;
138
+
139
+ int _sampleRate = 44100;
140
+ int _channels = 1;
141
+ };
142
+
143
+ #endif /* VROPCMRingBuffer_h */
@@ -72,9 +72,18 @@ public:
72
72
  _brushThickness = thickness;
73
73
  }
74
74
 
75
+ /*
76
+ When enabled, lines are depth-tested against the scene (occluded by closer surfaces).
77
+ Default false to preserve the original overlay behavior.
78
+ */
79
+ void setDepthTestEnabled(bool enabled) {
80
+ _depthTestEnabled = enabled;
81
+ }
82
+
75
83
  private:
76
84
  std::vector<std::vector<VROVector3f>> _paths;
77
85
  float _brushThickness;
86
+ bool _depthTestEnabled = false;
78
87
  };
79
88
 
80
89
  #endif
@@ -208,6 +208,9 @@ public:
208
208
  return _depthTexture;
209
209
  }
210
210
 
211
+ void setDepthIsMonocular(bool monocular) { _depthIsMonocular = monocular; }
212
+ bool isDepthMonocular() const { return _depthIsMonocular; }
213
+
211
214
  void setSceneDepthTexture(std::shared_ptr<VROTexture> texture) {
212
215
  _sceneDepthTexture = texture;
213
216
  }
@@ -358,6 +361,7 @@ private:
358
361
  Depth texture from AR framework for occlusion.
359
362
  */
360
363
  std::shared_ptr<VROTexture> _depthTexture;
364
+ bool _depthIsMonocular = false;
361
365
 
362
366
  /*
363
367
  Scene depth texture captured from the previous frame's base render pass.