@reactvision/react-viro 2.55.0 → 2.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/android/react_viro/react_viro-release.aar +0 -0
- package/android/viro_renderer/viro_renderer-release.aar +0 -0
- package/components/AR/ViroARScene.tsx +17 -0
- package/components/AR/ViroARSceneNavigator.tsx +33 -0
- package/components/Utilities/StreamingAudioManager.ts +73 -0
- package/components/Utilities/ViroVersion.ts +1 -1
- package/components/ViroCameraTexture.tsx +230 -0
- package/components/ViroGameLoop.tsx +74 -0
- package/components/ViroGameLoopUtils.ts +52 -0
- package/components/ViroObjectDetector.tsx +222 -0
- package/components/ViroVirtualButton.tsx +101 -0
- package/components/ViroVirtualJoystick.tsx +126 -0
- package/components/hooks/useGameLoop.ts +78 -0
- package/dist/components/AR/ViroARScene.d.ts +12 -0
- package/dist/components/AR/ViroARScene.js +8 -1
- package/dist/components/AR/ViroARSceneNavigator.d.ts +30 -0
- package/dist/components/Utilities/StreamingAudioManager.d.ts +22 -0
- package/dist/components/Utilities/StreamingAudioManager.js +45 -0
- package/dist/components/Utilities/ViroVersion.d.ts +1 -1
- package/dist/components/Utilities/ViroVersion.js +1 -1
- package/dist/components/ViroCameraTexture.d.ts +137 -0
- package/dist/components/ViroCameraTexture.js +170 -0
- package/dist/components/ViroGameLoop.d.ts +31 -0
- package/dist/components/ViroGameLoop.js +31 -0
- package/dist/components/ViroGameLoopUtils.d.ts +26 -0
- package/dist/components/ViroGameLoopUtils.js +44 -0
- package/dist/components/ViroObjectDetector.d.ts +138 -0
- package/dist/components/ViroObjectDetector.js +85 -0
- package/dist/components/ViroVirtualButton.d.ts +76 -0
- package/dist/components/ViroVirtualButton.js +62 -0
- package/dist/components/ViroVirtualJoystick.d.ts +85 -0
- package/dist/components/ViroVirtualJoystick.js +72 -0
- package/dist/components/hooks/useGameLoop.d.ts +34 -0
- package/dist/components/hooks/useGameLoop.js +63 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.js +20 -2
- package/dist/plugins/withViro.js +2 -15
- package/dist/plugins/withViroAndroid.js +52 -15
- package/dist/plugins/withViroIos.js +0 -6
- package/index.ts +21 -0
- package/ios/ViroReact.podspec +25 -6
- package/ios/dist/ViroRenderer/ViroKit.framework/ARCoreCoreMLSemanticsResources.bundle/Info.plist +0 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/ARCoreResources.bundle/Info.plist +0 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARDepthMesh.h +9 -1
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrame.h +33 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrameiOS.h +26 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARScene.h +15 -1
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSceneDelegateiOS.h +8 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSession.h +7 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSessioniOS.h +18 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARWorldMesh.h +111 -8
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROAudioPlayer.h +19 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROAudioPlayerStreamiOS.h +95 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROCameraTextureiOS.h +17 -1
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODriver.h +20 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODynamicGeometry.h +170 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRODynamicMeshNode.h +87 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROGLTFLoader.h +5 -1
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROGameLoopListener.h +125 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROInputState.h +133 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROMonocularDepthEstimator.h +73 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROObjectDetector.h +134 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROPCMRingBuffer.h +143 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROPencil.h +9 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRORenderContext.h +4 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VRORenderer.h +1 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROVertexBuffer.h +102 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROViewAR.h +9 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROVirtualControllerRegistry.h +106 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Info.plist +0 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/Shaders.dat +1 -1
- package/ios/dist/ViroRenderer/ViroKit.framework/ViroKit +0 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/constant_fsh.glsl +1 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/constant_fsh_adreno.glsl +1 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/standard_fsh.glsl +1 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/standard_vsh.glsl +4 -0
- package/ios/dist/ViroRenderer/ViroKit.framework/standard_vsh_adreno.glsl +4 -0
- package/ios/dist/ViroRendererVisionOS/ViroKit.podspec +27 -0
- package/ios/dist/include/VRTARScene.h +1 -0
- package/ios/dist/include/VRTARSceneNavigator.h +3 -0
- package/ios/dist/include/VRTCameraTexture.h +62 -0
- package/ios/dist/include/VRTCameraTextureManager.h +30 -0
- package/ios/dist/include/VRTCameraTextureModule.h +38 -0
- package/ios/dist/include/VRTGameLoopView.h +19 -0
- package/ios/dist/include/VRTGameLoopViewManager.h +8 -0
- package/ios/dist/include/VRTManagedAnimation.h +67 -0
- package/ios/dist/include/VRTStreamingAudioModule.h +21 -0
- package/ios/dist/include/VRTVirtualButtonView.h +47 -0
- package/ios/dist/include/VRTVirtualButtonViewManager.h +21 -0
- package/ios/dist/include/VRTVirtualJoystickView.h +59 -0
- package/ios/dist/include/VRTVirtualJoystickViewManager.h +26 -0
- package/ios/dist/lib/libViroReact.a +0 -0
- package/package.json +17 -20
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
//
|
|
2
|
+
// VROGameLoopListener.h
|
|
3
|
+
// ViroKit
|
|
4
|
+
//
|
|
5
|
+
// Copyright © 2026 ReactVision. All rights reserved.
|
|
6
|
+
//
|
|
7
|
+
// Permission is hereby granted, free of charge, to any person obtaining
|
|
8
|
+
// a copy of this software and associated documentation files (the
|
|
9
|
+
// "Software"), to deal in the Software without restriction, including
|
|
10
|
+
// without limitation the rights to use, copy, modify, merge, publish,
|
|
11
|
+
// distribute, sublicense, and/or sell copies of the Software, and to
|
|
12
|
+
// permit persons to whom the Software is furnished to do so, subject to
|
|
13
|
+
// the following conditions:
|
|
14
|
+
//
|
|
15
|
+
// The above copyright notice and this permission notice shall be included
|
|
16
|
+
// in all copies or substantial portions of the Software.
|
|
17
|
+
|
|
18
|
+
#ifndef VROGameLoopListener_h
|
|
19
|
+
#define VROGameLoopListener_h
|
|
20
|
+
|
|
21
|
+
#include <functional>
|
|
22
|
+
#include "VROFrameListener.h"
|
|
23
|
+
#include "VROTime.h"
|
|
24
|
+
|
|
25
|
+
/*
|
|
26
|
+
VROGameLoopListener — thin VROFrameListener wrapper that tracks delta time and
|
|
27
|
+
elapsed time, then fires typed callbacks from the render thread.
|
|
28
|
+
|
|
29
|
+
Typical use (C++, from native bridge code):
|
|
30
|
+
|
|
31
|
+
auto loop = std::make_shared<VROGameLoopListener>();
|
|
32
|
+
loop->setOnFrameWillRender([](float dt, float elapsed) {
|
|
33
|
+
// game logic here — runs before physics and rendering
|
|
34
|
+
});
|
|
35
|
+
frameSynchronizer->addFrameListener(loop);
|
|
36
|
+
|
|
37
|
+
For fixed-step simulation (F5 addendum):
|
|
38
|
+
|
|
39
|
+
loop->setFixedHz(30);
|
|
40
|
+
loop->setOnFixedStep([](float dt) {
|
|
41
|
+
// guaranteed 1/30 s ticks
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
Thread safety: callbacks are invoked on the render thread. Callers are
|
|
45
|
+
responsible for marshalling to another thread if needed.
|
|
46
|
+
*/
|
|
47
|
+
class VROGameLoopListener : public VROFrameListener {
|
|
48
|
+
public:
|
|
49
|
+
using UpdateCallback = std::function<void(float dtSec, float elapsedSec)>;
|
|
50
|
+
using FixedStepCallback = std::function<void(float dtSec)>;
|
|
51
|
+
|
|
52
|
+
VROGameLoopListener() = default;
|
|
53
|
+
~VROGameLoopListener() override = default;
|
|
54
|
+
|
|
55
|
+
// ── Variable-step callbacks ────────────────────────────────────────────────
|
|
56
|
+
|
|
57
|
+
void setOnFrameWillRender(UpdateCallback cb) { _onWillRender = std::move(cb); }
|
|
58
|
+
void setOnFrameDidRender(UpdateCallback cb) { _onDidRender = std::move(cb); }
|
|
59
|
+
|
|
60
|
+
// ── Fixed-step (F5 addendum) ───────────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
/*
|
|
63
|
+
Set the fixed simulation frequency in Hz (e.g. 30 for libsm64, 60 for typical
|
|
64
|
+
physics). Each onFrameWillRender call drains the accumulator, firing the fixed-
|
|
65
|
+
step callback as many times as needed to stay in sync with wall time.
|
|
66
|
+
A clamped dt prevents a "spiral of death" when frames are very slow.
|
|
67
|
+
*/
|
|
68
|
+
void setFixedHz(float hz) {
|
|
69
|
+
_fixedHz = hz;
|
|
70
|
+
_fixedDt = (hz > 0.f) ? 1.f / hz : 0.f;
|
|
71
|
+
_accumulator = 0.f;
|
|
72
|
+
_fixedEnabled = (hz > 0.f);
|
|
73
|
+
}
|
|
74
|
+
void setOnFixedStep(FixedStepCallback cb) { _onFixedStep = std::move(cb); }
|
|
75
|
+
|
|
76
|
+
// ── VROFrameListener ──────────────────────────────────────────────────────
|
|
77
|
+
|
|
78
|
+
void onFrameWillRender(const VRORenderContext &context) override {
|
|
79
|
+
double now = VROTimeCurrentMillis();
|
|
80
|
+
if (_startTime < 0.0) _startTime = _lastTime = now;
|
|
81
|
+
|
|
82
|
+
float dt = (float)((now - _lastTime) / 1000.0);
|
|
83
|
+
float elapsed = (float)((now - _startTime) / 1000.0);
|
|
84
|
+
_lastTime = now;
|
|
85
|
+
|
|
86
|
+
// Fixed-step drain — clamp frame dt to avoid spiral of death
|
|
87
|
+
if (_fixedEnabled && _onFixedStep && _fixedDt > 0.f) {
|
|
88
|
+
float safeDt = (dt < kMaxFrameDt) ? dt : kMaxFrameDt;
|
|
89
|
+
_accumulator += safeDt;
|
|
90
|
+
while (_accumulator >= _fixedDt) {
|
|
91
|
+
_onFixedStep(_fixedDt);
|
|
92
|
+
_accumulator -= _fixedDt;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (_onWillRender) _onWillRender(dt, elapsed);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
void onFrameDidRender(const VRORenderContext &context) override {
|
|
100
|
+
if (!_onDidRender) return;
|
|
101
|
+
double now = VROTimeCurrentMillis();
|
|
102
|
+
float dt = (float)((now - _lastTime) / 1000.0);
|
|
103
|
+
float elapsed = (float)((now - _startTime) / 1000.0);
|
|
104
|
+
_onDidRender(dt, elapsed);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private:
|
|
108
|
+
// Maximum frame delta clamped in fixed-step mode (prevents spiral of death
|
|
109
|
+
// if the app is paused or debugging halts the render loop).
|
|
110
|
+
static constexpr float kMaxFrameDt = 0.25f; // 4 fps floor
|
|
111
|
+
|
|
112
|
+
UpdateCallback _onWillRender;
|
|
113
|
+
UpdateCallback _onDidRender;
|
|
114
|
+
FixedStepCallback _onFixedStep;
|
|
115
|
+
|
|
116
|
+
double _startTime = -1.0;
|
|
117
|
+
double _lastTime = -1.0;
|
|
118
|
+
|
|
119
|
+
bool _fixedEnabled = false;
|
|
120
|
+
float _fixedHz = 0.f;
|
|
121
|
+
float _fixedDt = 0.f;
|
|
122
|
+
float _accumulator = 0.f;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
#endif /* VROGameLoopListener_h */
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
//
|
|
2
|
+
// VROInputState.h
|
|
3
|
+
// ViroKit
|
|
4
|
+
//
|
|
5
|
+
// Copyright © 2026 ReactVision. All rights reserved.
|
|
6
|
+
//
|
|
7
|
+
// Permission is hereby granted, free of charge, to any person obtaining
|
|
8
|
+
// a copy of this software and associated documentation files (the
|
|
9
|
+
// "Software"), to deal in the Software without restriction, including
|
|
10
|
+
// without limitation the rights to use, copy, modify, merge, publish,
|
|
11
|
+
// distribute, sublicense, and/or sell copies of the Software, and to
|
|
12
|
+
// permit persons to whom the Software is furnished to do so, subject to
|
|
13
|
+
// the following conditions:
|
|
14
|
+
//
|
|
15
|
+
// The above copyright notice and this permission notice shall be included
|
|
16
|
+
// in all copies or substantial portions of the Software.
|
|
17
|
+
|
|
18
|
+
#ifndef VROInputState_h
|
|
19
|
+
#define VROInputState_h
|
|
20
|
+
|
|
21
|
+
#include <stdint.h>
|
|
22
|
+
#include <mutex>
|
|
23
|
+
|
|
24
|
+
/*
|
|
25
|
+
Unified controller state for game-style continuous input. Filled by zero or more
|
|
26
|
+
source adapters (virtual touch joystick, MFi/Bluetooth gamepad, device tilt,
|
|
27
|
+
voice, hand-tracking) and read by consumers (game loop, simulation adapter,
|
|
28
|
+
native modules forwarding to JS).
|
|
29
|
+
|
|
30
|
+
Independent of VROInputControllerBase / VROInputPresenter, which are oriented at
|
|
31
|
+
pointer/click events for raycasting in AR/VR scenes. VROInputState models the
|
|
32
|
+
"continuous controller snapshot" use case: per-frame stick deflections, button
|
|
33
|
+
hold states, analog triggers.
|
|
34
|
+
|
|
35
|
+
Thread-safety: write methods are safe to call from any thread (touch handler,
|
|
36
|
+
gamepad callback, sensor thread). Readers call snapshot() to get a consistent
|
|
37
|
+
view of the entire state in one shot. Internal mutex contention is negligible
|
|
38
|
+
because each writer touches the state at most a few times per frame.
|
|
39
|
+
|
|
40
|
+
Button indices are caller-defined; the table below documents the recommended
|
|
41
|
+
default mapping for game-style apps. Adapters can map their physical inputs
|
|
42
|
+
to any index they like.
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
class VROInputState {
|
|
46
|
+
|
|
47
|
+
public:
|
|
48
|
+
|
|
49
|
+
static constexpr int kMaxButtons = 32;
|
|
50
|
+
|
|
51
|
+
/*
|
|
52
|
+
Recommended button-index mapping for SM64-style inputs and most casual
|
|
53
|
+
games. Adapters that target this layout populate buttons at these indices;
|
|
54
|
+
consumers that read this layout know what each bit means. Mappings are
|
|
55
|
+
not enforced — they are conventions.
|
|
56
|
+
*/
|
|
57
|
+
enum DefaultButton : int {
|
|
58
|
+
Button_A = 0, // jump / primary action
|
|
59
|
+
Button_B = 1, // attack / grab / secondary action
|
|
60
|
+
Button_X = 2,
|
|
61
|
+
Button_Y = 3,
|
|
62
|
+
Button_Z = 4, // crouch / ground-pound
|
|
63
|
+
Button_L1 = 5,
|
|
64
|
+
Button_R1 = 6,
|
|
65
|
+
Button_L2 = 7,
|
|
66
|
+
Button_R2 = 8,
|
|
67
|
+
Button_Start = 9,
|
|
68
|
+
Button_Select = 10,
|
|
69
|
+
Button_StickLClick = 11,
|
|
70
|
+
Button_StickRClick = 12,
|
|
71
|
+
Button_DPad_Up = 13,
|
|
72
|
+
Button_DPad_Down = 14,
|
|
73
|
+
Button_DPad_Left = 15,
|
|
74
|
+
Button_DPad_Right = 16,
|
|
75
|
+
// 17..31 free for app-specific buttons
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/*
|
|
79
|
+
Plain-data snapshot returned by snapshot() and consumed by readers. Safe
|
|
80
|
+
to copy and pass across threads.
|
|
81
|
+
*/
|
|
82
|
+
struct Snapshot {
|
|
83
|
+
float stickLX = 0.f;
|
|
84
|
+
float stickLY = 0.f;
|
|
85
|
+
float stickRX = 0.f;
|
|
86
|
+
float stickRY = 0.f;
|
|
87
|
+
float triggerL = 0.f;
|
|
88
|
+
float triggerR = 0.f;
|
|
89
|
+
uint32_t buttonBits = 0; // bit i = button i state (1 = pressed)
|
|
90
|
+
|
|
91
|
+
bool getButton(int idx) const {
|
|
92
|
+
if (idx < 0 || idx >= kMaxButtons) return false;
|
|
93
|
+
return (buttonBits & (1u << idx)) != 0;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
VROInputState() = default;
|
|
98
|
+
|
|
99
|
+
/*
|
|
100
|
+
Reader API. Returns a consistent snapshot of every field. Cheap; takes the
|
|
101
|
+
mutex once.
|
|
102
|
+
*/
|
|
103
|
+
Snapshot snapshot() const;
|
|
104
|
+
|
|
105
|
+
/*
|
|
106
|
+
Writer API. Each call takes the mutex briefly. Adapters can call these
|
|
107
|
+
from any thread.
|
|
108
|
+
|
|
109
|
+
Stick axis convention: x in [-1, 1] left-to-right, y in [-1, 1]
|
|
110
|
+
down-to-up. Adapters are responsible for sign conventions and clamping;
|
|
111
|
+
values outside [-1, 1] are accepted but consumers may behave oddly.
|
|
112
|
+
*/
|
|
113
|
+
void setStickL(float x, float y);
|
|
114
|
+
void setStickR(float x, float y);
|
|
115
|
+
void setTriggerL(float value);
|
|
116
|
+
void setTriggerR(float value);
|
|
117
|
+
void setButton(int idx, bool pressed);
|
|
118
|
+
|
|
119
|
+
/*
|
|
120
|
+
Reset all fields to neutral (sticks zeroed, triggers zeroed, no buttons
|
|
121
|
+
pressed). Called when a source adapter unregisters or when a controller
|
|
122
|
+
is intentionally cleared (e.g. on app background).
|
|
123
|
+
*/
|
|
124
|
+
void reset();
|
|
125
|
+
|
|
126
|
+
private:
|
|
127
|
+
|
|
128
|
+
mutable std::mutex _mtx;
|
|
129
|
+
Snapshot _state;
|
|
130
|
+
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
#endif /* VROInputState_h */
|
|
@@ -82,6 +82,15 @@ public:
|
|
|
82
82
|
* estimator->update(arFrame);
|
|
83
83
|
* auto depthTexture = estimator->getDepthTexture();
|
|
84
84
|
*/
|
|
85
|
+
/**
|
|
86
|
+
* Controls how the monocular depth estimator derives metric scale.
|
|
87
|
+
*/
|
|
88
|
+
enum class VROMonocularDepthCalibration {
|
|
89
|
+
None, // no scaling (raw model output, scale = 1.0)
|
|
90
|
+
Manual, // caller sets scale via setScaleFactor() (default)
|
|
91
|
+
LiDARReference, // stub: sample LiDAR to auto-derive scale (falls back to Manual)
|
|
92
|
+
};
|
|
93
|
+
|
|
85
94
|
class API_AVAILABLE(ios(14.0)) VROMonocularDepthEstimator :
|
|
86
95
|
public std::enable_shared_from_this<VROMonocularDepthEstimator> {
|
|
87
96
|
|
|
@@ -126,6 +135,16 @@ public:
|
|
|
126
135
|
*/
|
|
127
136
|
void update(const VROARFrame *frame);
|
|
128
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Run a single throwaway inference on a blank image to force CoreML / Neural Engine
|
|
140
|
+
* model specialization up front. Without this, the FIRST real inference pays the
|
|
141
|
+
* full ANE compile cost (many seconds on A-series chips) — the dominant cause of the
|
|
142
|
+
* long delay before the first depth frame appears. Safe to call once right after
|
|
143
|
+
* initWithModel(); runs asynchronously on the depth queue and does not touch the
|
|
144
|
+
* live depth buffers (it uses a private throwaway request).
|
|
145
|
+
*/
|
|
146
|
+
void warmup();
|
|
147
|
+
|
|
129
148
|
#pragma mark - Depth Output
|
|
130
149
|
|
|
131
150
|
/**
|
|
@@ -161,11 +180,28 @@ public:
|
|
|
161
180
|
* Set the scale factor for converting model output to metric depth.
|
|
162
181
|
* Some models output relative/disparity depth that needs scaling.
|
|
163
182
|
* Default is 1.0 (assumes model outputs metric depth in meters).
|
|
183
|
+
* Implicitly sets calibration mode to Manual.
|
|
164
184
|
*
|
|
165
185
|
* @param scale The scale factor to multiply depth values by.
|
|
166
186
|
*/
|
|
167
187
|
void setScaleFactor(float scale);
|
|
168
188
|
|
|
189
|
+
/**
|
|
190
|
+
* Set the calibration mode that controls how depth scale is derived.
|
|
191
|
+
* - None: raw model output (scale = 1.0)
|
|
192
|
+
* - Manual: caller-supplied scale via setScaleFactor() (default)
|
|
193
|
+
* - LiDARReference: stub, logs warning and falls back to Manual
|
|
194
|
+
*/
|
|
195
|
+
void setCalibrationMode(VROMonocularDepthCalibration mode);
|
|
196
|
+
VROMonocularDepthCalibration getCalibrationMode() const;
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Flush any pending depth data to the GPU texture.
|
|
200
|
+
* Must be called on the render (GL) thread. VROARSessioniOS calls this
|
|
201
|
+
* automatically each frame before depth is consumed.
|
|
202
|
+
*/
|
|
203
|
+
void flushPendingDepthUpdate();
|
|
204
|
+
|
|
169
205
|
/**
|
|
170
206
|
* Enable or disable temporal filtering for depth stability.
|
|
171
207
|
* When enabled, depth values are smoothed across frames using
|
|
@@ -232,6 +268,36 @@ public:
|
|
|
232
268
|
*/
|
|
233
269
|
int getDepthBufferHeight() const { return _depthHeight; }
|
|
234
270
|
|
|
271
|
+
/**
|
|
272
|
+
* Atomically snapshot the depth and confidence buffers for offline processing
|
|
273
|
+
* (e.g. world-mesh generation). Thread-safe — takes a copy under the depth mutex.
|
|
274
|
+
* Returns false if no data is available.
|
|
275
|
+
*/
|
|
276
|
+
bool snapshotDepthBuffers(std::vector<float> &outDepth,
|
|
277
|
+
std::vector<float> &outConfidence,
|
|
278
|
+
int &outWidth, int &outHeight) const;
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Sample the synthesized per-pixel confidence at depth-texture UV coordinates.
|
|
282
|
+
* Returns a value in [0, 1]: 1 = highly stable, 0 = discontinuity or invalid.
|
|
283
|
+
* Returns -1 if no confidence data is available (temporal filtering disabled or
|
|
284
|
+
* first frame). Thread-safe.
|
|
285
|
+
*
|
|
286
|
+
* @param u Depth texture U coordinate [0, 1].
|
|
287
|
+
* @param v Depth texture V coordinate [0, 1].
|
|
288
|
+
*/
|
|
289
|
+
float sampleConfidenceAtDepthUV(float u, float v) const;
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Set the confidence threshold for hit-test upgrade decisions on the monocular path.
|
|
293
|
+
* Hit results are upgraded to DepthPoint only when synthesized confidence exceeds
|
|
294
|
+
* this value. Default is 0.3.
|
|
295
|
+
*
|
|
296
|
+
* @param threshold Confidence threshold in [0, 1].
|
|
297
|
+
*/
|
|
298
|
+
void setHitTestConfidenceThreshold(float threshold);
|
|
299
|
+
float getHitTestConfidenceThreshold() const;
|
|
300
|
+
|
|
235
301
|
private:
|
|
236
302
|
// Graphics driver
|
|
237
303
|
std::weak_ptr<VRODriver> _driver;
|
|
@@ -258,17 +324,24 @@ private:
|
|
|
258
324
|
std::shared_ptr<VROTexture> _currentDepthTexture;
|
|
259
325
|
std::vector<float> _depthBuffer;
|
|
260
326
|
std::vector<float> _previousDepthBuffer;
|
|
327
|
+
std::vector<float> _confidenceBuffer; // per-pixel synthesized confidence [0,1]
|
|
261
328
|
int _depthWidth;
|
|
262
329
|
int _depthHeight;
|
|
263
330
|
VROMatrix4f _depthTextureTransform;
|
|
264
331
|
|
|
265
332
|
// Configuration
|
|
266
333
|
float _depthScaleFactor;
|
|
334
|
+
VROMonocularDepthCalibration _calibrationMode;
|
|
335
|
+
float _hitTestConfidenceThreshold;
|
|
267
336
|
bool _temporalFilteringEnabled;
|
|
268
337
|
float _temporalFilterAlpha;
|
|
269
338
|
int _targetFPS;
|
|
270
339
|
double _lastInferenceTime;
|
|
271
340
|
|
|
341
|
+
// W4: staging buffer for in-place GPU texture updates
|
|
342
|
+
std::vector<float> _stagingDepthBuffer;
|
|
343
|
+
std::atomic<bool> _stagingDirty;
|
|
344
|
+
|
|
272
345
|
// Diagnostics
|
|
273
346
|
float _currentFPS;
|
|
274
347
|
float _averageLatencyMs;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
//
|
|
2
|
+
// VROObjectDetector.h
|
|
3
|
+
// ViroRenderer
|
|
4
|
+
//
|
|
5
|
+
// Copyright © 2026 ReactVision. All rights reserved.
|
|
6
|
+
//
|
|
7
|
+
// Permission is hereby granted, free of charge, to any person obtaining
|
|
8
|
+
// a copy of this software and associated documentation files (the
|
|
9
|
+
// "Software"), to deal in the Software without restriction, including
|
|
10
|
+
// without limitation the rights to use, copy, modify, merge, publish,
|
|
11
|
+
// distribute, sublicense, and/or sell copies of the Software, and to
|
|
12
|
+
// permit persons to whom the Software is furnished to do so, subject to
|
|
13
|
+
// the following conditions:
|
|
14
|
+
//
|
|
15
|
+
// The above copyright notice and this permission notice shall be included
|
|
16
|
+
// in all copies or substantial portions of the Software.
|
|
17
|
+
//
|
|
18
|
+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
19
|
+
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
20
|
+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
21
|
+
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
22
|
+
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
23
|
+
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
24
|
+
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
25
|
+
|
|
26
|
+
#ifndef VROObjectDetector_h
|
|
27
|
+
#define VROObjectDetector_h
|
|
28
|
+
|
|
29
|
+
#include <string>
|
|
30
|
+
#include <vector>
|
|
31
|
+
#include <functional>
|
|
32
|
+
#include <memory>
|
|
33
|
+
|
|
34
|
+
/*
|
|
35
|
+
Bounding box in normalized image space [0, 1].
|
|
36
|
+
Origin is top-left corner of the image.
|
|
37
|
+
*/
|
|
38
|
+
struct VRODetectionBoundingBox {
|
|
39
|
+
float x; // left edge
|
|
40
|
+
float y; // top edge
|
|
41
|
+
float width;
|
|
42
|
+
float height;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
struct VRODetectedObject {
|
|
46
|
+
std::string label;
|
|
47
|
+
float confidence;
|
|
48
|
+
VRODetectionBoundingBox boundingBox;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/*
|
|
52
|
+
Inference mode matching YOLOE's three prompting strategies:
|
|
53
|
+
- PromptFree: LRPC head, 4,585 built-in RAM++ categories, no runtime prompt needed.
|
|
54
|
+
- Text: RepRTA, detect arbitrary categories given a list of text labels.
|
|
55
|
+
- Visual: SAVPE, detect objects similar to a reference image crop.
|
|
56
|
+
*/
|
|
57
|
+
enum class VRODetectorMode {
|
|
58
|
+
PromptFree,
|
|
59
|
+
Text,
|
|
60
|
+
Visual,
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
using VRODetectionCallback = std::function<void(const std::vector<VRODetectedObject> &)>;
|
|
64
|
+
|
|
65
|
+
/*
|
|
66
|
+
Abstract base class for on-device object detectors.
|
|
67
|
+
|
|
68
|
+
Platform implementations (iOS: CoreML / ONNX Runtime, Android: ONNX Runtime)
|
|
69
|
+
subclass this and implement loadModel() and detectInBuffer(). The bridge layer
|
|
70
|
+
owns the camera pipeline and calls detectInBuffer() on each sampled frame.
|
|
71
|
+
*/
|
|
72
|
+
class VROObjectDetector {
|
|
73
|
+
public:
|
|
74
|
+
virtual ~VROObjectDetector() {}
|
|
75
|
+
|
|
76
|
+
/*
|
|
77
|
+
Load the model from the given path.
|
|
78
|
+
On iOS: path to a .mlpackage (CoreML) or .onnx file.
|
|
79
|
+
On Android: path to a .onnx file inside the app's assets or files dir.
|
|
80
|
+
Returns true on success.
|
|
81
|
+
*/
|
|
82
|
+
virtual bool loadModel(const std::string &modelPath) = 0;
|
|
83
|
+
|
|
84
|
+
/*
|
|
85
|
+
Run detection on a raw RGB pixel buffer.
|
|
86
|
+
- pixels: pointer to contiguous RGB888 data (row-major)
|
|
87
|
+
- width / height: image dimensions in pixels
|
|
88
|
+
Results are delivered asynchronously via the detection callback.
|
|
89
|
+
*/
|
|
90
|
+
virtual void detectInBuffer(const uint8_t *pixels, int width, int height) = 0;
|
|
91
|
+
|
|
92
|
+
// --- configuration (call before loadModel or between frames) ---
|
|
93
|
+
|
|
94
|
+
void setMode(VRODetectorMode mode) {
|
|
95
|
+
_mode = mode;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/*
|
|
99
|
+
Text categories for Text mode (e.g. {"chair", "person", "laptop"}).
|
|
100
|
+
Ignored in PromptFree and Visual modes.
|
|
101
|
+
*/
|
|
102
|
+
void setCategories(const std::vector<std::string> &categories) {
|
|
103
|
+
_categories = categories;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
void setConfidenceThreshold(float threshold) {
|
|
107
|
+
_confidenceThreshold = threshold;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/*
|
|
111
|
+
NMS IoU threshold used during post-processing.
|
|
112
|
+
*/
|
|
113
|
+
void setIouThreshold(float threshold) {
|
|
114
|
+
_iouThreshold = threshold;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
void setCallback(VRODetectionCallback callback) {
|
|
118
|
+
_callback = callback;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
VRODetectorMode getMode() const { return _mode; }
|
|
122
|
+
float getConfidenceThreshold() const { return _confidenceThreshold; }
|
|
123
|
+
float getIouThreshold() const { return _iouThreshold; }
|
|
124
|
+
const std::vector<std::string> &getCategories() const { return _categories; }
|
|
125
|
+
|
|
126
|
+
protected:
|
|
127
|
+
VRODetectorMode _mode = VRODetectorMode::PromptFree;
|
|
128
|
+
std::vector<std::string> _categories;
|
|
129
|
+
float _confidenceThreshold = 0.4f;
|
|
130
|
+
float _iouThreshold = 0.45f;
|
|
131
|
+
VRODetectionCallback _callback;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
#endif /* VROObjectDetector_h */
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
//
|
|
2
|
+
// VROPCMRingBuffer.h
|
|
3
|
+
// ViroRenderer
|
|
4
|
+
//
|
|
5
|
+
// Copyright © 2026 ReactVision. All rights reserved.
|
|
6
|
+
//
|
|
7
|
+
// Permission is hereby granted, free of charge, to any person obtaining
|
|
8
|
+
// a copy of this software and associated documentation files (the
|
|
9
|
+
// "Software"), to deal in the Software without restriction, including
|
|
10
|
+
// without limitation the rights to use, copy, modify, merge, publish,
|
|
11
|
+
// distribute, sublicense, and/or sell copies of the Software, and to
|
|
12
|
+
// permit persons to whom the Software is furnished to do so, subject to
|
|
13
|
+
// the following conditions:
|
|
14
|
+
//
|
|
15
|
+
// The above copyright notice and this permission notice shall be included
|
|
16
|
+
// in all copies or substantial portions of the Software.
|
|
17
|
+
|
|
18
|
+
#ifndef VROPCMRingBuffer_h
|
|
19
|
+
#define VROPCMRingBuffer_h
|
|
20
|
+
|
|
21
|
+
#include <vector>
|
|
22
|
+
#include <atomic>
|
|
23
|
+
#include <algorithm>
|
|
24
|
+
#include <cstring>
|
|
25
|
+
|
|
26
|
+
/*
|
|
27
|
+
Lock-free single-producer / single-consumer ring buffer for interleaved float PCM
|
|
28
|
+
samples. Designed for the streaming audio path:
|
|
29
|
+
|
|
30
|
+
Producer (any thread): calls write() with chunks of samples from the
|
|
31
|
+
simulation / TTS / synthesiser.
|
|
32
|
+
|
|
33
|
+
Consumer (audio callback, real-time thread): calls read() to fill the OS
|
|
34
|
+
audio buffer. Must never block — silence (zeros) is returned when the
|
|
35
|
+
ring is empty.
|
|
36
|
+
|
|
37
|
+
Capacity is set at construction and is a power of two so that the index wrap
|
|
38
|
+
is a bitwise AND rather than a modulo. A capacity of 8192 samples at 32 kHz
|
|
39
|
+
stereo gives ~128 ms of headroom, sufficient to absorb a typical JS bridge
|
|
40
|
+
round-trip.
|
|
41
|
+
|
|
42
|
+
NOT safe for multiple producers or multiple consumers.
|
|
43
|
+
*/
|
|
44
|
+
class VROPCMRingBuffer {
|
|
45
|
+
|
|
46
|
+
public:
|
|
47
|
+
|
|
48
|
+
explicit VROPCMRingBuffer(size_t capacitySamples = 8192)
|
|
49
|
+
: _capacity(nextPow2(capacitySamples)),
|
|
50
|
+
_mask(_capacity - 1),
|
|
51
|
+
_buffer(_capacity, 0.0f),
|
|
52
|
+
_writePos(0),
|
|
53
|
+
_readPos(0) {}
|
|
54
|
+
|
|
55
|
+
/*
|
|
56
|
+
Write interleaved PCM samples. Called from the producer thread.
|
|
57
|
+
Returns the number of samples actually written (may be less than count
|
|
58
|
+
if the ring is nearly full — caller should retry or drop).
|
|
59
|
+
*/
|
|
60
|
+
size_t write(const float *data, size_t count) {
|
|
61
|
+
size_t w = _writePos.load(std::memory_order_relaxed);
|
|
62
|
+
size_t r = _readPos.load(std::memory_order_acquire);
|
|
63
|
+
size_t available = _capacity - (w - r); // free slots
|
|
64
|
+
size_t toWrite = std::min(count, available);
|
|
65
|
+
if (toWrite == 0) return 0;
|
|
66
|
+
|
|
67
|
+
size_t idx = w & _mask;
|
|
68
|
+
size_t firstPart = std::min(toWrite, _capacity - idx);
|
|
69
|
+
memcpy(_buffer.data() + idx, data, firstPart * sizeof(float));
|
|
70
|
+
if (toWrite > firstPart) {
|
|
71
|
+
memcpy(_buffer.data(), data + firstPart,
|
|
72
|
+
(toWrite - firstPart) * sizeof(float));
|
|
73
|
+
}
|
|
74
|
+
_writePos.store(w + toWrite, std::memory_order_release);
|
|
75
|
+
return toWrite;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/*
|
|
79
|
+
Read interleaved PCM samples into out. Called from the audio callback
|
|
80
|
+
(consumer thread). Fills with silence if fewer samples are available
|
|
81
|
+
than requested. Returns number of valid samples read.
|
|
82
|
+
*/
|
|
83
|
+
size_t read(float *out, size_t count) {
|
|
84
|
+
size_t r = _readPos.load(std::memory_order_relaxed);
|
|
85
|
+
size_t w = _writePos.load(std::memory_order_acquire);
|
|
86
|
+
size_t available = w - r;
|
|
87
|
+
size_t toRead = std::min(count, available);
|
|
88
|
+
|
|
89
|
+
if (toRead > 0) {
|
|
90
|
+
size_t idx = r & _mask;
|
|
91
|
+
size_t firstPart = std::min(toRead, _capacity - idx);
|
|
92
|
+
memcpy(out, _buffer.data() + idx, firstPart * sizeof(float));
|
|
93
|
+
if (toRead > firstPart) {
|
|
94
|
+
memcpy(out + firstPart, _buffer.data(),
|
|
95
|
+
(toRead - firstPart) * sizeof(float));
|
|
96
|
+
}
|
|
97
|
+
_readPos.store(r + toRead, std::memory_order_release);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Zero-fill any unfulfilled frames (silence on underrun).
|
|
101
|
+
if (toRead < count) {
|
|
102
|
+
memset(out + toRead, 0, (count - toRead) * sizeof(float));
|
|
103
|
+
}
|
|
104
|
+
return toRead;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
int sampleRate() const { return _sampleRate; }
|
|
108
|
+
int channels() const { return _channels; }
|
|
109
|
+
|
|
110
|
+
void setSampleRate(int sr) { _sampleRate = sr; }
|
|
111
|
+
void setChannels(int ch) { _channels = ch; }
|
|
112
|
+
|
|
113
|
+
size_t capacity() const { return _capacity; }
|
|
114
|
+
|
|
115
|
+
/*
|
|
116
|
+
Approximate number of samples currently buffered. Informational only —
|
|
117
|
+
may be slightly stale by the time the caller acts on it.
|
|
118
|
+
*/
|
|
119
|
+
size_t available() const {
|
|
120
|
+
return _writePos.load(std::memory_order_relaxed) -
|
|
121
|
+
_readPos.load(std::memory_order_relaxed);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private:
|
|
125
|
+
|
|
126
|
+
static size_t nextPow2(size_t n) {
|
|
127
|
+
size_t p = 1;
|
|
128
|
+
while (p < n) p <<= 1;
|
|
129
|
+
return p;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const size_t _capacity;
|
|
133
|
+
const size_t _mask;
|
|
134
|
+
std::vector<float> _buffer;
|
|
135
|
+
|
|
136
|
+
std::atomic<size_t> _writePos;
|
|
137
|
+
std::atomic<size_t> _readPos;
|
|
138
|
+
|
|
139
|
+
int _sampleRate = 44100;
|
|
140
|
+
int _channels = 1;
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
#endif /* VROPCMRingBuffer_h */
|
|
@@ -72,9 +72,18 @@ public:
|
|
|
72
72
|
_brushThickness = thickness;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
/*
|
|
76
|
+
When enabled, lines are depth-tested against the scene (occluded by closer surfaces).
|
|
77
|
+
Default false to preserve the original overlay behavior.
|
|
78
|
+
*/
|
|
79
|
+
void setDepthTestEnabled(bool enabled) {
|
|
80
|
+
_depthTestEnabled = enabled;
|
|
81
|
+
}
|
|
82
|
+
|
|
75
83
|
private:
|
|
76
84
|
std::vector<std::vector<VROVector3f>> _paths;
|
|
77
85
|
float _brushThickness;
|
|
86
|
+
bool _depthTestEnabled = false;
|
|
78
87
|
};
|
|
79
88
|
|
|
80
89
|
#endif
|
|
@@ -208,6 +208,9 @@ public:
|
|
|
208
208
|
return _depthTexture;
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
void setDepthIsMonocular(bool monocular) { _depthIsMonocular = monocular; }
|
|
212
|
+
bool isDepthMonocular() const { return _depthIsMonocular; }
|
|
213
|
+
|
|
211
214
|
void setSceneDepthTexture(std::shared_ptr<VROTexture> texture) {
|
|
212
215
|
_sceneDepthTexture = texture;
|
|
213
216
|
}
|
|
@@ -358,6 +361,7 @@ private:
|
|
|
358
361
|
Depth texture from AR framework for occlusion.
|
|
359
362
|
*/
|
|
360
363
|
std::shared_ptr<VROTexture> _depthTexture;
|
|
364
|
+
bool _depthIsMonocular = false;
|
|
361
365
|
|
|
362
366
|
/*
|
|
363
367
|
Scene depth texture captured from the previous frame's base render pass.
|