@babylonjs/react-native 0.4.0-alpha.8 → 0.64.0-alpha.48

package/android/build.gradle

@@ -19,6 +19,10 @@ def safeExtGet(prop, fallback) {
     rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
 }
 
+def reactProperties = new Properties()
+file("$projectDir/../../../react-native/ReactAndroid/gradle.properties").withInputStream { reactProperties.load(it) }
+def REACT_VERSION = reactProperties.getProperty("VERSION_NAME").split("\\.")[1].toInteger()
+
 apply plugin: 'com.android.library'
 apply plugin: 'maven'
 
@@ -59,6 +63,14 @@ android {
     }
 }
 
+// The full/real version of libturbomodulejsijni.so will be built and included in apps using React Native 0.64 or newer,
+// so exclude Babylon React Native's minimal version of these libs in this case.
+if (REACT_VERSION >= 64) {
+    android.packagingOptions.excludes += 'lib/armeabi-v7a/libturbomodulejsijni.so'
+    android.packagingOptions.excludes += 'lib/arm64-v8a/libturbomodulejsijni.so'
+    android.packagingOptions.excludes += 'lib/x86/libturbomodulejsijni.so'
+}
+
 repositories {
     // ref: https://www.baeldung.com/maven-local-repository
     mavenLocal()
@@ -77,7 +89,7 @@ repositories {
 dependencies {
     //noinspection GradleDynamicVersion
     implementation 'com.facebook.react:react-native:+'  // From node_modules
-    implementation 'com.google.ar:core:1.14.0'
+    implementation 'com.google.ar:core:1.22.0'
 }
 
 def configureReactNativePom(def pom) {

package/android/include/IXrContextARCore.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <arcore_c_api.h>
+
+typedef struct IXrContextARCore
+{
+    virtual bool IsInitialized() const = 0;
+    virtual ArSession* XrSession() const = 0;
+    virtual ArFrame* XrFrame() const = 0;
+} IXrContextARCore;

package/android/src/main/java/com/babylonreactnative/BabylonModule.java

@@ -27,9 +27,11 @@ public final class BabylonModule extends ReactContextBaseJavaModule {
         });
     }
 
-    // NOTE: This happens during dev mode reload, when the JS engine is being shutdown and restarted.
-    @Override
-    public void onCatalystInstanceDestroy() {
-        this.getReactApplicationContext().runOnJSQueueThread(BabylonNativeInterop::deinitialize);
+    @ReactMethod
+    public void resetView(Promise promise) {
+        this.getReactApplicationContext().runOnUiQueueThread(() -> {
+            BabylonNativeInterop.resetView();
+            promise.resolve(null);
+        });
     }
 }

package/android/src/main/java/com/babylonreactnative/BabylonNativeInterop.java

@@ -19,11 +19,14 @@ public final class BabylonNativeInterop {
         }
 
         public static native void initialize(Context context, long jsiRuntimeRef, CallInvokerHolder jsCallInvokerHolder);
-        public static native void deinitialize();
         public static native void setCurrentActivity(Activity activity);
         public static native void pause();
         public static native void resume();
         public static native void updateView(Surface surface);
+        public static native void renderView();
+        public static native void resetView();
+        public static native void updateXRView(Surface surface);
+        public static native boolean isXRActive();
         public static native void setTouchButtonState(int pointerId, boolean isDown, int x, int y);
         public static native void setTouchPosition(int pointerId, int x, int y);
     }
@@ -54,7 +57,6 @@ public final class BabylonNativeInterop {
 
             @Override
             public void onHostDestroy() {
-                BabylonNative.deinitialize();
             }
         };
 
@@ -79,14 +81,26 @@ public final class BabylonNativeInterop {
         reactContext.addActivityEventListener(BabylonNativeInterop.activityEventListener);
     }
 
-    public static void deinitialize() {
-        BabylonNative.deinitialize();
-    }
-
     public static void updateView(Surface surface) {
         BabylonNative.updateView(surface);
     }
 
+    public static void renderView() {
+        BabylonNative.renderView();
+    }
+
+    public static void resetView() {
+        BabylonNative.resetView();
+    }
+
+    public static void updateXRView(Surface surface) {
+        BabylonNative.updateXRView(surface);
+    }
+
+    public static boolean isXRActive() {
+        return BabylonNative.isXRActive();
+    }
+
     public static void reportMotionEvent(MotionEvent motionEvent) {
         int maskedAction = motionEvent.getActionMasked();
         boolean isPointerDown = maskedAction == MotionEvent.ACTION_DOWN || maskedAction == MotionEvent.ACTION_POINTER_DOWN;

package/android/src/main/java/com/babylonreactnative/EngineView.java

@@ -2,15 +2,21 @@ package com.babylonreactnative;
 
 import android.annotation.TargetApi;
 import android.graphics.Bitmap;
+import android.graphics.SurfaceTexture;
 import android.os.Build;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.util.Base64;
 import android.view.MotionEvent;
 import android.view.PixelCopy;
+import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
+import android.view.TextureView;
 import android.view.View;
+import android.widget.FrameLayout;
+
+import androidx.annotation.NonNull;
 
 import com.facebook.react.bridge.ReactContext;
 import com.facebook.react.uimanager.UIManagerModule;
@@ -18,19 +24,83 @@ import com.facebook.react.uimanager.events.EventDispatcher;
 
 import java.io.ByteArrayOutputStream;
 
-public final class EngineView extends SurfaceView implements SurfaceHolder.Callback, View.OnTouchListener {
+public final class EngineView extends FrameLayout implements SurfaceHolder.Callback, TextureView.SurfaceTextureListener, View.OnTouchListener {
+    private static final FrameLayout.LayoutParams childViewLayoutParams = new FrameLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT);
+    private TextureView transparentTextureView;
+    private Surface transparentSurface = null;
+    private SurfaceView opaqueSurfaceView = null;
+    private SurfaceView xrSurfaceView;
     private final EventDispatcher reactEventDispatcher;
+    private Runnable renderRunnable;
 
     public EngineView(ReactContext reactContext) {
         super(reactContext);
-        this.getHolder().addCallback(this);
+
+        this.setIsTransparent(false);
+
+        this.xrSurfaceView = new SurfaceView(reactContext);
+        this.xrSurfaceView.setLayoutParams(childViewLayoutParams);
+        this.xrSurfaceView.getHolder().addCallback(new SurfaceHolder.Callback() {
+            @Override
+            public void surfaceCreated(SurfaceHolder holder) {
+                // surfaceChanged is also called when the surface is created, so just do all the handling there
+            }
+
+            @Override
+            public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                BabylonNativeInterop.updateXRView(holder.getSurface());
+            }
+
+            @Override
+            public void surfaceDestroyed(SurfaceHolder holder) {
+                BabylonNativeInterop.updateXRView(null);
+            }
+        });
+        this.xrSurfaceView.setVisibility(View.INVISIBLE);
+        this.addView(this.xrSurfaceView);
+
         this.setOnTouchListener(this);
+
         this.reactEventDispatcher = reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher();
     }
 
+    // ------------------------------------
+    // TextureView related
+
+    public void setIsTransparent(Boolean isTransparent) {
+        if (isTransparent) {
+            if (this.opaqueSurfaceView != null) {
+                this.opaqueSurfaceView.setVisibility(View.GONE);
+                this.opaqueSurfaceView = null;
+            }
+            if (this.transparentTextureView == null) {
+                this.transparentTextureView = new TextureView(this.getContext());
+                this.transparentTextureView.setLayoutParams(EngineView.childViewLayoutParams);
+                this.transparentTextureView.setSurfaceTextureListener(this);
+                this.transparentTextureView.setOpaque(false);
+                this.addView(this.transparentTextureView);
+            }
+        } else {
+            if (this.transparentTextureView != null) {
+                this.transparentTextureView.setVisibility(View.GONE);
+                this.transparentTextureView = null;
+            }
+            if (this.opaqueSurfaceView == null) {
+                this.opaqueSurfaceView = new SurfaceView(this.getContext());
+                this.opaqueSurfaceView.setLayoutParams(EngineView.childViewLayoutParams);
+                this.opaqueSurfaceView.getHolder().addCallback(this);
+                this.addView(this.opaqueSurfaceView);
+            }
+        }
+        // xr view needs to be on top of views that might be created after it.
+        if (this.xrSurfaceView != null) {
+            this.xrSurfaceView.bringToFront();
+        }
+    }
+
     @Override
     public void surfaceCreated(SurfaceHolder surfaceHolder) {
-        // surfaceChanged is also called when the surface is created, so just do all the handling there
+        this.startRenderLoop();
     }
 
     @Override
@@ -40,8 +110,47 @@ public final class EngineView extends SurfaceView implements SurfaceHolder.Callb
 
     @Override
     public void surfaceDestroyed(SurfaceHolder surfaceHolder) {
+        this.removeCallbacks(this.renderRunnable);
+        this.renderRunnable = null;
+    }
+
+    @Override
+    public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surfaceTexture, int i, int i1) {
+        this.startRenderLoop();
+        this.acquireNewTransparentSurface(surfaceTexture);
+        BabylonNativeInterop.updateView(this.transparentSurface);
     }
 
+    @Override
+    public void onSurfaceTextureSizeChanged(@NonNull SurfaceTexture surfaceTexture, int i, int i1) {
+        this.acquireNewTransparentSurface(surfaceTexture);
+        BabylonNativeInterop.updateView(this.transparentSurface);
+    }
+
+    @Override
+    public boolean onSurfaceTextureDestroyed(@NonNull SurfaceTexture surfaceTexture) {
+        this.stopRenderLoop();
+        this.transparentSurface.release();
+        this.transparentSurface = null;
+        return false;
+    }
+
+    @Override
+    public void onSurfaceTextureUpdated(@NonNull SurfaceTexture surfaceTexture) {
+        this.acquireNewTransparentSurface(surfaceTexture);
+        BabylonNativeInterop.updateView(this.transparentSurface);
+    }
+
+    private void acquireNewTransparentSurface(@NonNull SurfaceTexture surfaceTexture) {
+        if (this.transparentSurface != null) {
+            this.transparentSurface.release();
+        }
+        this.transparentSurface = new Surface(surfaceTexture);
+    }
+
+    // ------------------------------------
+    // Common
+
     @Override
     public boolean onTouch(View view, MotionEvent motionEvent) {
         BabylonNativeInterop.reportMotionEvent(motionEvent);
@@ -67,21 +176,55 @@ public final class EngineView extends SurfaceView implements SurfaceHolder.Callb
         helperThread.start();
         final Handler helperThreadHandler = new Handler(helperThread.getLooper());
 
-        // Request the pixel copy.
-        PixelCopy.request(this, bitmap, (copyResult) ->  {
+        Surface sourceSurface = this.transparentSurface;
+        if (BabylonNativeInterop.isXRActive()) {
+            sourceSurface = this.xrSurfaceView.getHolder().getSurface();
+        } else if (this.opaqueSurfaceView != null) {
+            sourceSurface = this.opaqueSurfaceView.getHolder().getSurface();
+        }
+        PixelCopy.request(sourceSurface, bitmap, getOnPixelCopyFinishedListener(bitmap, helperThread), helperThreadHandler);
+    }
+
+    // ---------------------------------------------------------------------------------------------
+    // Returns the listener for the PixelCopy.request function call
+    @NonNull
+    private PixelCopy.OnPixelCopyFinishedListener getOnPixelCopyFinishedListener(Bitmap bitmap, HandlerThread helperThread) {
+        return (copyResult) -> {
             // If the pixel copy was a success then convert the image to a base 64 encoded jpeg and fire the event.
             String encoded = "";
             if (copyResult == PixelCopy.SUCCESS) {
                 ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream();
-                bitmap.compress(Bitmap.CompressFormat.JPEG, 100, byteArrayStream);
+                bitmap.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayStream);
                 byte[] byteArray = byteArrayStream.toByteArray();
                 bitmap.recycle();
                 encoded = Base64.encodeToString(byteArray, Base64.DEFAULT);
             }
-
             SnapshotDataReturnedEvent snapshotEvent = new SnapshotDataReturnedEvent(this.getId(), encoded);
             reactEventDispatcher.dispatchEvent(snapshotEvent);
             helperThread.quitSafely();
-        }, helperThreadHandler);
+        };
+    }
+
+    private void startRenderLoop() {
+        if(this.renderRunnable == null){
+            this.renderRunnable = new Runnable() {
+                @Override
+                public void run() {
+                    if (BabylonNativeInterop.isXRActive()) {
+                        EngineView.this.xrSurfaceView.setVisibility(View.VISIBLE);
+                    } else {
+                        EngineView.this.xrSurfaceView.setVisibility(View.INVISIBLE);
+                    }
+                    BabylonNativeInterop.renderView();
+                    EngineView.this.postOnAnimation(this);
+                }
+            };
+            this.postOnAnimation(this.renderRunnable);
+        }
+    }
+
+    private void stopRenderLoop() {
+        this.removeCallbacks(this.renderRunnable);
+        this.renderRunnable = null;
     }
 }

package/android/src/main/java/com/babylonreactnative/EngineViewManager.java

@@ -1,5 +1,7 @@
 package com.babylonreactnative;
 
+import android.widget.FrameLayout;
+
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 
@@ -7,6 +9,7 @@ import com.facebook.react.bridge.ReadableArray;
 import com.facebook.react.common.MapBuilder;
 import com.facebook.react.uimanager.SimpleViewManager;
 import com.facebook.react.uimanager.ThemedReactContext;
+import com.facebook.react.uimanager.annotations.ReactProp;
 
 import java.util.Map;
 
@@ -20,6 +23,11 @@ public final class EngineViewManager extends SimpleViewManager<EngineView> {
         return "EngineView";
     }
 
+    @ReactProp(name = "isTransparent")
+    public void setIsTransparent(EngineView view, Boolean isTransparent) {
+        view.setIsTransparent(isTransparent);
+    }
+
     @NonNull
     @Override
     protected EngineView createViewInstance(@NonNull ThemedReactContext reactContext) {

package/{index.ts → index.d.ts}

@@ -1,3 +1,4 @@
 export * from './EngineView';
 export * from './EngineHook';
 export * from './NativeCapture';
+export * from './FontFace';

package/index.js

@@ -0,0 +1,5 @@
+export * from './EngineView';
+export * from './EngineHook';
+export * from './NativeCapture';
+export * from './FontFace';
+//# sourceMappingURL=index.js.map

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../Modules/@babylonjs/react-native/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC"}

package/ios/BabylonModule.mm

@@ -25,4 +25,11 @@ RCT_EXPORT_METHOD(initialize:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseR
     });
 }
 
+RCT_EXPORT_METHOD(resetView:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) {
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [BabylonNativeInterop resetView];
+        resolve([NSNull null]);
+    });
+}
+
 @end

package/ios/BabylonNativeInterop.h

@@ -6,5 +6,9 @@
 @interface BabylonNativeInterop : NSObject
 + (void)initialize:(RCTBridge*)bridge;
 + (void)updateView:(MTKView*)mtkView;
++ (void)renderView;
++ (void)resetView;
++ (void)updateXRView:(MTKView*)mtkView;
++ (bool)isXRActive;
 + (void)reportTouchEvent:(MTKView*)mtkView touches:(NSSet<UITouch*>*)touches event:(UIEvent*)event;
 @end

package/ios/BabylonNativeInterop.mm

@@ -24,7 +24,7 @@ namespace {
 
 @implementation BabylonNativeInterop
 
-static NSMutableArray* activeTouches;
+static NSMutableArray* activeTouches = [NSMutableArray new];
 
 + (void)initialize:(RCTBridge*)bridge {
     auto jsCallInvoker{ bridge.jsCallInvoker };
@@ -36,7 +36,7 @@ static NSMutableArray* activeTouches;
         });
     } };
 
-    Babylon::Initialize(*GetJSIRuntime(bridge), std::move(jsDispatcher));
+    BabylonNative::Initialize(*GetJSIRuntime(bridge), std::move(jsDispatcher));
 
     [[NSNotificationCenter defaultCenter] removeObserver:self
         name:RCTBridgeWillInvalidateModulesNotification
@@ -51,49 +51,72 @@ static NSMutableArray* activeTouches;
 // NOTE: This happens during dev mode reload, when the JS engine is being shutdown and restarted.
 + (void)onBridgeWillInvalidate:(NSNotification*)notification
 {
-    Babylon::Deinitialize();
+    BabylonNative::Deinitialize();
 }
 
 + (void)updateView:(MTKView*)mtkView {
-    const int width = static_cast<int>(mtkView.bounds.size.width * UIScreen.mainScreen.scale);
-    const int height = static_cast<int>(mtkView.bounds.size.height * UIScreen.mainScreen.scale);
+    const CGFloat scale = mtkView.contentScaleFactor;
+    const int width = static_cast<int>(mtkView.bounds.size.width * scale);
+    const int height = static_cast<int>(mtkView.bounds.size.height * scale);
     if (width != 0 && height != 0) {
-        Babylon::UpdateView((__bridge void*)mtkView, width, height);
+        BabylonNative::UpdateView(mtkView, width, height);
     }
 }
 
++ (void)renderView {
+    BabylonNative::RenderView();
+}
+
++ (void)resetView {
+    BabylonNative::ResetView();
+}
+
++ (void)updateXRView:(MTKView*)mtkView {
+    BabylonNative::UpdateXRView(mtkView);
+}
+
++ (bool)isXRActive {
+    return BabylonNative::IsXRActive();
+}
+
 + (void)reportTouchEvent:(MTKView*)mtkView touches:(NSSet<UITouch*>*)touches event:(UIEvent*)event {
     for (UITouch* touch in touches) {
         if (touch.view == mtkView) {
-            const CGFloat scale = UIScreen.mainScreen.scale;
+            const CGFloat scale = mtkView.contentScaleFactor;
             const CGPoint pointerPosition = [touch locationInView:mtkView];
             const uint32_t x = static_cast<uint32_t>(pointerPosition.x * scale);
             const uint32_t y = static_cast<uint32_t>(pointerPosition.y * scale);
 
             switch (touch.phase) {
                 case UITouchPhaseBegan: {
+                    // The activeTouches array only grows, it does not shrink (to keep indices constant since they are used as pointer ids),
+                    // so look for an unused (null) array element and reuse it if found. Otherwise, add a new entry to the array.
                     NSUInteger pointerId = [activeTouches indexOfObject:[NSNull null]];
-                    if (pointerId == NSNotFound) {
+                    if (pointerId != NSNotFound) {
+                        [activeTouches replaceObjectAtIndex:pointerId withObject:touch];
+                    } else {
                         pointerId = [activeTouches count];
                         [activeTouches addObject:touch];
-                    } else {
-                        [activeTouches replaceObjectAtIndex:pointerId withObject:touch];
                     }
-                    Babylon::SetTouchButtonState(static_cast<uint32_t>(pointerId), true, x, y);
+                    BabylonNative::SetTouchButtonState(static_cast<uint32_t>(pointerId), true, x, y);
                     break;
                 }
 
                 case UITouchPhaseMoved: {
                     NSUInteger pointerId = [activeTouches indexOfObject:touch];
-                    Babylon::SetTouchPosition(static_cast<uint32_t>(pointerId), x, y);
+                    if (pointerId != NSNotFound) {
+                        BabylonNative::SetTouchPosition(static_cast<uint32_t>(pointerId), x, y);
+                    }
                     break;
                 }
 
                 case UITouchPhaseEnded:
                 case UITouchPhaseCancelled: {
                     NSUInteger pointerId = [activeTouches indexOfObject:touch];
-                    [activeTouches replaceObjectAtIndex:pointerId withObject:[NSNull null]];
-                    Babylon::SetTouchButtonState(static_cast<uint32_t>(pointerId), false, x, y);
+                    if (pointerId != NSNotFound) {
+                        [activeTouches replaceObjectAtIndex:pointerId withObject:[NSNull null]];
+                        BabylonNative::SetTouchButtonState(static_cast<uint32_t>(pointerId), false, x, y);
+                    }
                     break;
                 }
 

package/ios/EngineViewManager.mm

@@ -11,17 +11,19 @@
 @interface EngineView : MTKView
 
 @property (nonatomic, copy) RCTDirectEventBlock onSnapshotDataReturned;
+@property (nonatomic, assign) BOOL isTransparent;
+
 
 @end
 
 @implementation EngineView {
-    RCTBridge* bridge;
+    const RCTBridge* bridge;
+    MTKView* xrView;
 }
 
 - (instancetype)init:(RCTBridge*)_bridge {
     if (self = [super initWithFrame:CGRectZero device:MTLCreateSystemDefaultDevice()]) {
         bridge = _bridge;
-
         super.translatesAutoresizingMaskIntoConstraints = false;
         super.colorPixelFormat = MTLPixelFormatBGRA8Unorm_sRGB;
         super.depthStencilPixelFormat = MTLPixelFormatDepth32Float;
@@ -29,6 +31,16 @@
     return self;
 }
 
+- (void)setIsTransparentFlag:(NSNumber*)isTransparentFlag {
+    BOOL isTransparent = [isTransparentFlag intValue] == 1;
+    if(isTransparent){
+        [self setOpaque:NO];
+    } else {
+        [self setOpaque:YES];
+    }
+    self.isTransparent = isTransparent;
+}
+
 - (void)setBounds:(CGRect)bounds {
     [super setBounds:bounds];
     [BabylonNativeInterop updateView:self];
@@ -50,6 +62,28 @@
     [BabylonNativeInterop reportTouchEvent:self touches:touches event:event];
 }
 
+- (void)drawRect:(CGRect)rect {
+    if ([BabylonNativeInterop isXRActive]) {
+        if (!xrView) {
+            xrView = [[MTKView alloc] initWithFrame:self.bounds device:self.device];
+            xrView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+            xrView.userInteractionEnabled = false;
+            [self addSubview:xrView];
+            [BabylonNativeInterop updateXRView:xrView];
+        }
+    } else if (xrView) {
+        [BabylonNativeInterop updateXRView:nil];
+        [xrView removeFromSuperview];
+        xrView = nil;
+    }
+
+    [BabylonNativeInterop renderView];
+}
+
+-(void)dealloc {
+    [BabylonNativeInterop updateXRView:nil];
+}
+
 - (void)takeSnapshot {
     // We must take the screenshot on the main thread otherwise we might fail to get a valid handle on the view's image.
     dispatch_async(dispatch_get_main_queue(), ^{
@@ -62,7 +96,7 @@
         // Grab the image from the graphics context, and convert into a base64 encoded JPG.
         UIImage* capturedImage = UIGraphicsGetImageFromCurrentImageContext();
         UIGraphicsEndImageContext();
-        NSData* jpgData = UIImageJPEGRepresentation(capturedImage, 1.0f);
+        NSData* jpgData = UIImageJPEGRepresentation(capturedImage, .8f);
         NSString* encodedData = [jpgData base64EncodedStringWithOptions:0];
         
         // Fire the onSnapshotDataReturned event if hooked up.
@@ -80,6 +114,10 @@
 
 @implementation EngineViewManager
 
+RCT_CUSTOM_VIEW_PROPERTY(isTransparent, NSNumber*, EngineView){
+    [view setIsTransparentFlag:json];
+}
+
 RCT_EXPORT_MODULE(EngineViewManager)
 
 RCT_EXPORT_VIEW_PROPERTY(onSnapshotDataReturned, RCTDirectEventBlock)