@reactvision/react-viro 2.55.0 → 2.57.0

package/dist/plugins/withViroAndroid.js

@@ -101,11 +101,6 @@ const withBranchAndroid = (config) => {
     ]);
 };
 const withViroProjectBuildGradle = (config) => (0, config_plugins_1.withProjectBuildGradle)(config, async (newConfig) => {
-    // Enforce New Architecture requirement
-    if (!newConfig.modResults.contents.includes("newArchEnabled=true")) {
-        config_plugins_1.WarningAggregator.addWarningAndroid("withViroAndroid", "ViroReact requires New Architecture to be enabled. " +
-            'Please add "newArchEnabled=true" to your android/gradle.properties file.');
-    }
     newConfig.modResults.contents = newConfig.modResults.contents.replace(/minSdkVersion.*/, `minSdkVersion = 24`);
     // Ensure New Architecture is enabled
     //if (!newConfig.modResults.contents.includes("newArchEnabled=true")) {
@@ -221,23 +216,37 @@ const withViroManifest = (config) => (0, config_plugins_1.withAndroidManifest)(c
             }
         }
     }
-    if (viroPluginConfig.includes("GVR") ||
-        viroPluginConfig.includes("OVR_MOBILE")) {
-        //   <!-- Add the following line for cardboard -->
-        //   <category android:name="com.google.intent.category.CARDBOARD" />
+    // Derive active XR modes from the config at apply-time.
+    // `viroPluginConfig` (module-level) is updated by withBranchAndroid's
+    // withDangerousMod callback, which runs *after* withAndroidManifest
+    // callbacks — so it is still ["AR","GVR"] here. Read from config.plugins
+    // directly instead.
+    const activeXrModes = (() => {
+        const p = (newConfig.plugins ?? []).find((plugin) => Array.isArray(plugin) && plugin[0] === "@reactvision/react-viro");
+        if (Array.isArray(p) && p[1]?.android?.xRMode) {
+            const xrMode = p[1].android.xRMode;
+            return Array.isArray(xrMode) ? xrMode : [xrMode];
+        }
+        return viroPluginConfig;
+    })();
+    if (activeXrModes.includes("GVR") ||
+        activeXrModes.includes("OVR_MOBILE")) {
         contents?.manifest?.application?.[0]?.activity[0]["intent-filter"][0].category.push({
             $: {
                 "android:name": "com.google.intent.category.CARDBOARD",
             },
         });
-        //   <!-- Add the following line for daydream -->
-        //   <category android:name="com.google.intent.category.DAYDREAM" />
         contents?.manifest?.application?.[0]?.activity[0]["intent-filter"][0].category.push({
             $: {
                 "android:name": "com.google.intent.category.DAYDREAM",
             },
         });
     }
+    // android.permission.SYSTEM_ALERT_WINDOW is merged in from React Native's
+    // debug manifest and is rejected by the Meta Quest Store.
+    if (contents.manifest["uses-permission"]) {
+        contents.manifest["uses-permission"] = contents.manifest["uses-permission"].filter((p) => p.$?.["android:name"] !== "android.permission.SYSTEM_ALERT_WINDOW");
+    }
     contents.manifest.queries = [
         {
             package: [
@@ -267,11 +276,13 @@ const withViroManifest = (config) => (0, config_plugins_1.withAndroidManifest)(c
             "tools:replace": "required",
         },
     });
+    // Keep GLES 3.0 declared (required=false) so the Quest Store validator
+    // sees a graphics API. Previously tools:node="remove" silently stripped
+    // this entry from the merged manifest entirely.
     contents.manifest["uses-feature"].push({
         $: {
             "android:glEsVersion": "0x00030000",
             "android:required": "false",
-            "tools:node": "remove",
             "tools:replace": "required",
         },
     });
@@ -290,7 +301,7 @@ const withViroManifest = (config) => (0, config_plugins_1.withAndroidManifest)(c
         },
     });
     // Quest-specific features and permissions — after uses-feature is initialized
-    if (viroPluginConfig.includes("QUEST")) {
+    if (activeXrModes.includes("QUEST")) {
         contents.manifest["uses-feature"].push({
             $: {
                 "android:name": "android.hardware.vr.headtracking",
@@ -313,6 +324,13 @@ const withViroManifest = (config) => (0, config_plugins_1.withAndroidManifest)(c
                 "android:required": "false",
             },
         });
+        // Required when com.oculus.permission.EYE_TRACKING is declared.
+        contents.manifest["uses-feature"].push({
+            $: {
+                "android:name": "oculus.software.eye_tracking",
+                "android:required": "false",
+            },
+        });
         const existingPermissions = (contents.manifest["uses-permission"] || [])
             .map((p) => p.$?.["android:name"]);
         if (!existingPermissions.includes("com.oculus.permission.HAND_TRACKING")) {
@@ -541,7 +559,12 @@ class VRActivity : ReactActivity() {
             return config;
         },
     ]);
-    // 2. Add VRActivity to AndroidManifest
+    // 2. Cap targetSdkVersion to 34 — Meta Quest Store rejects targetSdk > 34.
+    config = (0, config_plugins_1.withAppBuildGradle)(config, (config) => {
+        config.modResults.contents = config.modResults.contents.replace(/targetSdk(?:Version)?\s*[=\s]\s*(\d+)/g, (match, ver) => parseInt(ver, 10) > 34 ? match.replace(ver, "34") : match);
+        return config;
+    });
+    // 3. Add VRActivity to AndroidManifest
     config = (0, config_plugins_1.withAndroidManifest)(config, async (config) => {
         const app = config.modResults.manifest.application?.[0];
         if (!app)
@@ -566,10 +589,24 @@ class VRActivity : ReactActivity() {
                         ],
                     },
                 ],
+                "meta-data": [
+                    {
+                        $: {
+                            "android:name": "com.oculus.vr.focusaware",
+                            "android:value": "true",
+                        },
+                    },
+                ],
             });
         }
-        // Inject com.oculus.app_id into <application> for Meta Quest App Name
+        // Quest store validator requires all activities to be landscape.
+        // Only apply when targeting Quest (questAppId present); AR/phone apps use portrait.
         const questAppId = props?.android?.questAppId;
+        const mainActivity = app.activity?.[0];
+        if (questAppId && mainActivity?.$ && mainActivity.$["android:name"] !== ".VRActivity") {
+            mainActivity.$["android:screenOrientation"] = "landscape";
+        }
+        // Inject com.oculus.app_id into <application> for Meta Quest App Name
         if (questAppId) {
             if (!app["meta-data"])
                 app["meta-data"] = [];

package/dist/plugins/withViroIos.js

@@ -35,12 +35,6 @@ const withViroPods = (config) => {
                 }
             }
             fs_1.default.readFile(`${root}/Podfile`, "utf-8", (err, data) => {
-                // Check for New Architecture environment variable
-                if (!data.includes('ENV["RCT_NEW_ARCH_ENABLED"]') &&
-                    !data.includes("RCT_NEW_ARCH_ENABLED=1")) {
-                    config_plugins_1.WarningAggregator.addWarningIOS("withViroIos", "ViroReact requires New Architecture to be enabled. " +
-                        "Please set RCT_NEW_ARCH_ENABLED=1 in your ios/.xcode.env file.");
-                }
                 // ViroReact with integrated Fabric support
                 let viroPods = `  # ViroReact with integrated New Architecture (Fabric) support\n` +
                     `  # Automatically includes Fabric components when RCT_NEW_ARCH_ENABLED=1\n` +

package/index.ts

@@ -28,6 +28,11 @@ import {
   ViroTrackingStateConstants,
 } from "./components/ViroConstants";
 import { ViroController } from "./components/ViroController";
+import { ViroVirtualJoystick } from "./components/ViroVirtualJoystick";
+import { ViroVirtualButton } from "./components/ViroVirtualButton";
+import { ViroGameLoop } from "./components/ViroGameLoop";
+import { ViroGameLoopUtils } from "./components/ViroGameLoopUtils";
+import { useGameLoop, useLateUpdate, useFixedUpdate } from "./components/hooks/useGameLoop";
 import { ViroDirectionalLight } from "./components/ViroDirectionalLight";
 import { ViroFlexView } from "./components/ViroFlexView";
 import { ViroGeometry } from "./components/ViroGeometry";
@@ -41,6 +46,10 @@ import {
   ViroShaderModifier,
 } from "./components/Material/ViroMaterials";
 import { ViroMaterialVideo } from "./components/ViroMaterialVideo";
+import { ViroCameraTexture } from "./components/ViroCameraTexture";
+export type { ViroCameraPosition, ViroCameraReadyEvent } from "./components/ViroCameraTexture";
+import { ViroObjectDetector } from "./components/ViroObjectDetector";
+export type { ViroDetectorMode, ViroDetectedObject, ViroDetectionBoundingBox, ViroDetectionEvent, ViroDetectorReadyEvent, ViroDetectorErrorEvent } from "./components/ViroObjectDetector";
 import { ViroNode } from "./components/ViroNode";
 import { ViroOmniLight } from "./components/ViroOmniLight";
 import { ViroOrbitCamera } from "./components/ViroOrbitCamera";
@@ -180,6 +189,7 @@ import { ViroQuestEntryPoint } from "./components/ViroQuestEntryPoint";
 import { VRQuestNavigatorBridge } from "./components/Utilities/VRQuestNavigatorBridge";
 import { VRModuleOpenXR, useVRViewTag, exitVRScene } from "./components/Utilities/VRModuleOpenXR";
 import type { VRModuleOpenXRType } from "./components/Utilities/VRModuleOpenXR";
+import { StreamingAudioManager } from "./components/Utilities/StreamingAudioManager";
 import { AppRegistry } from "react-native";
 
 // Auto-register the Quest VR entry point. VRActivity launches this component
@@ -199,6 +209,13 @@ export {
   ViroButton,
   ViroCamera,
   ViroController,
+  ViroVirtualJoystick,
+  ViroVirtualButton,
+  ViroGameLoop,
+  ViroGameLoopUtils,
+  useGameLoop,
+  useLateUpdate,
+  useFixedUpdate,
   ViroDirectionalLight,
   ViroFlexView,
   ViroGeometry,
@@ -207,6 +224,8 @@ export {
   ViroMaterials,
   ViroARCamera,
   ViroMaterialVideo,
+  ViroCameraTexture,
+  ViroObjectDetector,
   ViroNode,
   ViroOmniLight,
   ViroOrbitCamera,
@@ -244,6 +263,8 @@ export {
   useVRViewTag,
   exitVRScene,
   Viro3DSceneNavigator,
+  // Streaming audio
+  StreamingAudioManager,
   // Utilities
   hasOpenXRSupport,
   isQuest,

package/ios/ViroReact.podspec

@@ -18,16 +18,20 @@ Pod::Spec.new do |s|
   # visionOS: CompositorServices drives the immersive render loop.
   s.visionos.frameworks = ['Metal', 'MetalKit', 'CompositorServices', 'ARKit']
 
+  # iOS: frameworks required by source files compiled from the pod
+  # (VRTObjectDetectorView uses AVFoundation + Accelerate; CoreVideo for CVPixelBuffer)
+  s.ios.frameworks = ['AVFoundation', 'Accelerate', 'CoreVideo']
+
   # Base source files (always included)
   source_files_array = ['ViroReact/**/*.{h,m,mm,swift}']
   header_files_array = ['ViroReact/**/*.h']
-  
+
   # Include dist files if they exist (for release builds)
   if File.exist?(File.join(__dir__, 'dist/include'))
     source_files_array << 'dist/include/**/*.{h,m,mm}'
     header_files_array << 'dist/include/*.h'
   end
-  
+
   s.source_files        = source_files_array
   s.public_header_files = header_files_array
 
@@ -39,15 +43,30 @@ Pod::Spec.new do |s|
   if File.exist?(File.join(__dir__, 'dist/lib/libViroReact.a'))
     s.vendored_libraries = 'dist/lib/libViroReact.a'
   end
-  
+
   # React Native dependencies
   s.dependency 'React-Core'
-  
+
+  # ONNX Runtime is distributed as a vendored dynamic xcframework (onnxruntime.xcframework).
+  # When the xcframework is present in ios/dist/Frameworks/, enable inference by setting:
+  #   GCC_PREPROCESSOR_DEFINITIONS = $(inherited) VIRO_ONNXRUNTIME_AVAILABLE=1
+  # Until then, VRTObjectDetectorView compiles with the camera pipeline active
+  # and inference returning empty results.
+  if File.exist?(File.join(__dir__, 'dist/Frameworks/onnxruntime.xcframework'))
+    s.vendored_frameworks = [
+      'dist/ViroRenderer/ViroKit.framework',
+      'dist/Frameworks/onnxruntime.xcframework'
+    ]
+    s.pod_target_xcconfig = {
+      'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) RCT_NEW_ARCH_ENABLED=1 VIRO_ONNXRUNTIME_AVAILABLE=1'
+    }
+  end
+
   # Fabric dependencies
   s.dependency 'React-RCTFabric'
   s.dependency 'React-Fabric'
   s.dependency 'React-FabricComponents'
-  
+
   # Fabric-specific build configuration
   s.pod_target_xcconfig = {
     'SWIFT_VERSION' => '5.0',
@@ -63,5 +82,5 @@ Pod::Spec.new do |s|
     'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) RCT_NEW_ARCH_ENABLED=1',
     'OTHER_CPLUSPLUSFLAGS' => '$(inherited) -std=c++17'
   }
-  
+
 end

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARDepthMesh.h

@@ -28,6 +28,7 @@
 
 #include <vector>
 #include <memory>
+#include <string>
 #include "VROVector3f.h"
 
 /**
@@ -52,7 +53,8 @@ public:
      */
     VROARDepthMesh(std::vector<VROVector3f> vertices,
                    std::vector<int> indices,
-                   std::vector<float> confidences);
+                   std::vector<float> confidences,
+                   std::string source = "lidar");
 
     ~VROARDepthMesh();
 
@@ -91,10 +93,16 @@ public:
      */
     bool isValid() const;
 
+    /**
+     * Get the depth source identifier. "lidar" or "monocular".
+     */
+    const std::string& getSource() const { return _source; }
+
 private:
     std::vector<VROVector3f> _vertices;
     std::vector<int> _indices;
     std::vector<float> _confidences;
+    std::string _source;
 };
 
 #endif /* VROARDepthMesh_h */

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrame.h

@@ -176,6 +176,29 @@ public:
         return nullptr;
     }
 
+    /**
+     * Generate a persistent mesh from all active ARMeshAnchor objects (iOS LiDAR, iOS 13.4+).
+     * ARKit accumulates the mesh as the user scans the environment — geometry seen in
+     * previous frames remains present even when no longer in the camera frustum.
+     * Returns nullptr on non-LiDAR devices, Android, or unsupported OS versions.
+     * Source tag: "lidar"
+     */
+    virtual std::shared_ptr<VROARDepthMesh> generateMeshAnchorMesh() {
+        return nullptr;
+    }
+
+    /**
+     * Generate a mesh by triangulating all detected AR plane anchors.
+     * Fallback path for non-LiDAR iOS and non-Depth-API Android devices.
+     * Plane anchors are persistent within the AR session, so the mesh degrades
+     * gracefully rather than disappearing on unsupported hardware.
+     * Returns nullptr if no planes have been detected yet.
+     * Source tag: "plane"
+     */
+    virtual std::shared_ptr<VROARDepthMesh> generatePlaneMesh() {
+        return nullptr;
+    }
+
     /*
      Returns the transform matrix to convert from camera texture coordinates
      to depth texture coordinates. The depth map may have a different
@@ -188,6 +211,16 @@ public:
         return VROMatrix4f::identity();
     }
 
+    /*
+     Debug-only variant: orientation-corrected transform WITHOUT the ScaleFill
+     crop correction. Maps the full screen [0,1]×[0,1] to the full depth texture
+     [0,1]×[0,1] so the debug overlay fills the screen without magenta bands.
+     The occlusion path still uses getDepthTextureTransform() for correctness.
+     */
+    virtual VROMatrix4f getDepthDebugTextureTransform() const {
+        return VROMatrix4f::identity();
+    }
+
     // ========================================================================
     // Scene Semantics API
     // ========================================================================

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARFrameiOS.h

@@ -94,6 +94,17 @@ public:
      */
     VROMatrix4f getDepthTextureTransform() const;
 
+    /*
+     Returns a simplified depth transform for DEBUG visualization only.
+     Unlike getDepthTextureTransform(), this does NOT apply the ScaleFill
+     crop correction — it maps the full screen [0,1]×[0,1] to the full
+     depth texture [0,1]×[0,1]. This gives a single full-screen depth
+     overlay (no magenta bands at the edges of the coverage region).
+     The occlusion path still uses getDepthTextureTransform() for
+     correctness (only occlude where real depth data exists).
+     */
+    VROMatrix4f getDepthDebugTextureTransform() const;
+
     // Scene Semantics support (requires ARCore SDK with Semantics extension)
     float getSemanticLabelFraction(VROSemanticLabel label) override;
 
@@ -103,6 +114,12 @@ public:
         float minConfidence = 0.3f,
         float maxDepth = 5.0f) override;
 
+    // Persistent mesh from ARMeshAnchor (LiDAR, iOS 13.4+)
+    std::shared_ptr<VROARDepthMesh> generateMeshAnchorMesh() override;
+
+    // Plane-based fallback mesh (non-LiDAR / non-Depth-API)
+    std::shared_ptr<VROARDepthMesh> generatePlaneMesh() override;
+
 private:
 
     ARFrame *_frame;
@@ -127,6 +144,15 @@ private:
      */
     float sampleDepthTextureAtUV(std::shared_ptr<VROTexture> texture, float u, float v) const;
 
+    /*
+     Unproject a camera-image-space point (normalized [0,1] in ARKit's landscape image
+     space, the same space used for [ARFrame hitTest:]) at a given metric depth (meters
+     along the optical axis) to a world-space position. Returns false if camera
+     intrinsics are unavailable. Used to build a depth-derived DepthPoint hit result.
+     */
+    bool unprojectToWorld(VROVector3f cameraImagePoint, float depthMeters,
+                          VROVector3f &outWorld) const;
+
 };
 
 #endif

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARScene.h

@@ -44,6 +44,12 @@ public:
     virtual void onTrackingUpdated(VROARTrackingState state, VROARTrackingStateReason reason) = 0;
     virtual void onAmbientLightUpdate(float ambientLightIntensity, VROVector3f ambientLightColor) = 0;
     virtual void onWorldMeshUpdated(const VROWorldMeshStats& stats) {}
+
+    /*
+     Called once when depth data (LiDAR or monocular) first becomes available for the
+     session, i.e. hit tests can now return DepthPoints. Optional.
+     */
+    virtual void onDepthReady() {}
 };
 
 class VROARScene : public VROScene {
@@ -113,6 +119,12 @@ public:
     void setDelegate(std::shared_ptr<VROARSceneDelegate> delegate);
     void setTrackingState(VROARTrackingState state, VROARTrackingStateReason reason, bool force);
 
+    /*
+     Notify the delegate that depth data is now available. Fires the delegate's
+     onDepthReady() exactly once per scene; subsequent calls are no-ops.
+     */
+    void notifyDepthReady();
+
     /*
      Get the ambient light estimate of the scene.
      */
@@ -151,6 +163,7 @@ public:
     bool isWorldMeshEnabled() const;
     void setWorldMeshConfig(const VROWorldMeshConfig& config);
     VROWorldMeshStats getWorldMeshStats() const;
+    std::shared_ptr<VROARWorldMesh> getWorldMesh() const { return _worldMesh; }
 
     /*
      Override computePhysics to add world mesh debug drawing.
@@ -179,7 +192,8 @@ private:
     std::shared_ptr<VRONode> _pointCloudNode;
     std::shared_ptr<VROFixedParticleEmitter> _pointCloudEmitter;
     std::weak_ptr<VROARSceneDelegate> _delegate;
-    
+    bool _depthReadyNotified = false;
+
     /*
      Ambient light estimation. The intensity is in lumens and the color is in linear space.
      */

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSceneDelegateiOS.h

@@ -38,6 +38,8 @@
 - (void)onAnchorFound:(std::shared_ptr<VROARAnchor>)anchor;
 - (void)onAnchorUpdated:(std::shared_ptr<VROARAnchor>)anchor;
 - (void)onAnchorRemoved:(std::shared_ptr<VROARAnchor>)anchor;
+@optional
+- (void)onDepthReady;
 @end
 
 class VROARSceneDelegateiOS : public VROARSceneDelegate, public VROARDeclarativeSessionDelegate {
@@ -71,6 +73,12 @@ public:
         [_delegate onAmbientLightUpdate:intensity color:color];
     }
 
+    virtual void onDepthReady() {
+        if ([_delegate respondsToSelector:@selector(onDepthReady)]) {
+            [_delegate onDepthReady];
+        }
+    }
+
 private:
     __weak id<VROARSceneDelegateProtocol> _delegate;
 };

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSession.h

@@ -370,6 +370,13 @@ public:
         _occlusionMode = mode;
     }
 
+    /*
+     Notify the session that world mesh was enabled or disabled.
+     Platform implementations use this to activate depth sensing when the
+     world mesh needs it (e.g. ARCore must set DepthMode::Automatic).
+     */
+    virtual void onWorldMeshEnabled(bool enabled) {}
+
     /*
      Get the current occlusion mode.
      */

package/ios/dist/ViroRenderer/ViroKit.framework/Headers/VROARSessioniOS.h

@@ -126,6 +126,19 @@ public:
     void setPreferMonocularDepth(bool prefer);
     bool isPreferMonocularDepth() const;
 
+    // Use the front (TrueDepth) camera via ARFaceTrackingConfiguration.
+    // When true, world tracking, planes, and LiDAR are unavailable.
+    void setFrontCameraEnabled(bool enabled);
+    bool isFrontCameraEnabled() const { return _frontCameraEnabled; }
+
+    // Multiply all monocular depth values by this factor before use.
+    // 1.0 = no change (default). Use < 1.0 if model overestimates distance.
+    void setMonocularDepthScale(float scale);
+
+    // Set the target inference rate (default 5). Thermal state overrides this
+    // downward automatically (Fair→3fps, Serious→2fps, Critical→stop).
+    void setMonocularDepthTargetFPS(int fps);
+
     /*
      Set the base URL from which to download the depth model.
      The full URL will be: baseURL/DepthPro.mlmodelc.zip
@@ -224,6 +237,8 @@ public:
      Get the native ARSession object for anchor creation.
      */
     ARSession *getARSession() const { return _session; }
+    ARConfiguration *getSessionConfiguration() const { return _sessionConfiguration; }
+    bool isPaused() const { return _sessionPaused; }
 
 private:
     
@@ -356,7 +371,10 @@ private:
     std::shared_ptr<VROMonocularDepthEstimator> _monocularDepthEstimator;
     bool _monocularDepthEnabled;
     bool _preferMonocularDepth;  // When true, use monocular even on LiDAR devices
+    bool _frontCameraEnabled;    // When true, use ARFaceTrackingConfiguration (front camera)
     bool _monocularDepthLoading;
+    float _monocularDepthScale;  // Multiplied into depth values (1.0 = no change)
+    int   _monocularDepthTargetFPS;  // 0 = use estimator default
     std::shared_ptr<VRODriver> _driver;
 
     void updateTrackingType(VROTrackingType trackingType);