@webspatial/platform-visionos 1.0.4 → 1.1.0

package/web-spatial/view/SpatializedElementView.swift

@@ -0,0 +1,178 @@
+import SwiftUI
+
+// zIndex() have some bug, so use zOrderBias to simulate zIndex effect
+let zOrderBias = 0.001
+
+struct SpatializedElementView<Content: View>: View {
+    @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
+    @Environment(SpatialScene.self) var spatialScene: SpatialScene
+
+    var parentScrollOffset: Vec2
+    var content: Content
+
+    init(parentScrollOffset: Vec2, @ViewBuilder content: () -> Content) {
+        self.parentScrollOffset = parentScrollOffset
+        self.content = content()
+    }
+
+    // Begin Interaction
+    var gesture: some Gesture {
+        DragGesture()
+            .onChanged(onDragging)
+            .onEnded(onDraggingEnded)
+            .simultaneously(with:
+                RotateGesture3D()
+                    .onChanged(onRotateGesture3D)
+                    .onEnded(onRotateGesture3DEnd)
+            )
+            .simultaneously(with:
+                MagnifyGesture()
+                    .onChanged(onMagnifyGesture)
+                    .onEnded(onMagnifyGestureEnd)
+            )
+            .simultaneously(with:
+                SpatialTapGesture(count: 1)
+                    .onEnded(onTapEnded)
+            )
+    }
+
+    private func onRotateGesture3D(_ event: RotateGesture3D.Value) {
+        if spatializedElement.enableRotateGesture || spatializedElement.enableRotateStartGesture {
+            let gestureEvent = WebSpatialRotateGuestureEvent(
+                detail: .init(
+                    rotation: event.rotation,
+                    startAnchor3D: event.startAnchor3D,
+                    startLocation3D: event.startLocation3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onRotateGesture3DEnd(_ event: RotateGesture3D.Value) {
+        if spatializedElement.enableRotateEndGesture {
+            let gestureEvent = WebSpatialRotateEndGuestureEvent(
+                detail: .init(
+                    rotation: event.rotation,
+                    startAnchor3D: event.startAnchor3D,
+                    startLocation3D: event.startLocation3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onDragging(_ event: DragGesture.Value) {
+        if spatializedElement.enableDragStartGesture || spatializedElement.enableDragGesture {
+            let gestureEvent = WebSpatialDragGuestureEvent(detail: .init(
+                location3D: event.location3D,
+                startLocation3D: event.startLocation3D,
+                translation3D: event.translation3D,
+                predictedEndTranslation3D: event.predictedEndTranslation3D,
+                predictedEndLocation3D: event.predictedEndLocation3D,
+                velocity: event.velocity
+            ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onDraggingEnded(_ event: DragGesture.Value) {
+        if spatializedElement.enableDragEndGesture {
+            let gestureEvent = WebSpatialDragEndGuestureEvent(
+                detail: .init(
+                    location3D: event.location3D,
+                    startLocation3D: event.startLocation3D,
+                    translation3D: event.translation3D,
+                    predictedEndTranslation3D: event.predictedEndTranslation3D,
+                    predictedEndLocation3D: event.predictedEndLocation3D,
+                    velocity: event.velocity
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onTapEnded(_ event: SpatialTapGesture.Value) {
+        if spatializedElement.enableTapGesture {
+            spatialScene.sendWebMsg(spatializedElement.id, WebSpatialTapGuestureEvent(detail: .init(location3D: event.location3D)))
+        }
+    }
+
+    private func onMagnifyGesture(_ event: MagnifyGesture.Value) {
+        if spatializedElement.enableMagnifyGesture || spatializedElement.enableMagnifyStartGesture {
+            let gestureEvent = WebSpatialMagnifyGuestureEvent(
+                detail: .init(
+                    magnification: event.magnification,
+                    velocity: event.velocity,
+                    startLocation3D: event.startLocation3D,
+                    startAnchor3D: event.startAnchor3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onMagnifyGestureEnd(_ event: MagnifyGesture.Value) {
+        if spatializedElement.enableMagnifyEndGesture {
+            let gestureEvent = WebSpatialMagnifyEndGuestureEvent(
+                detail: .init(
+                    magnification: event.magnification,
+                    velocity: event.velocity,
+                    startLocation3D: event.startLocation3D,
+                    startAnchor3D: event.startAnchor3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    // End Interaction
+
+    @ViewBuilder
+    var body: some View {
+        let transform = spatializedElement.transform
+        let translation = transform.translation
+        let scale = transform.scale
+        let rotation = transform.rotation!
+
+        let width = spatializedElement.width
+        let height = spatializedElement.height
+        let depth = spatializedElement.depth
+        let anchor = spatializedElement.rotationAnchor
+
+        let centerX = spatializedElement.clientX - (spatializedElement.scrollWithParent ? parentScrollOffset.x : 0)
+        let centerY = spatializedElement.clientY - (spatializedElement.scrollWithParent ? parentScrollOffset.y : 0)
+
+        let opacity = spatializedElement.opacity
+        let visible = spatializedElement.visible
+        let enableGesture = spatializedElement.enableGesture
+
+        let z = translation.z + (spatializedElement.zIndex * zOrderBias)
+        let smallOffset = z == 0.0 ? 0.0001 : 0
+
+        content.simultaneousGesture(enableGesture ? gesture : nil)
+            .frame(width: width, height: height)
+            .frame(depth: depth, alignment: .back)
+            .onGeometryChange3D(for: AffineTransform3D.self) { proxy in
+                let rect3d = proxy.frame(in: .named("SpatialScene"))
+                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerClientCube(origin: rect3d.origin, size: rect3d.size))
+                return proxy.transform(in: .named("SpatialScene"))!
+            } action: { _, new in
+                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerTransform(detail: new))
+            }
+            .frame(depth: 0, alignment: .back)
+            // use .offset(smallVal) to workaround for glassEffect not working and small width/height spatialDiv not working
+            .offset(z: smallOffset)
+            .scaleEffect(
+                x: scale.width,
+                y: scale.height,
+                z: scale.depth,
+                anchor: anchor
+            )
+            .rotation3DEffect(
+                rotation,
+                anchor: anchor
+            )
+            .offset(x: translation.x, y: translation.y)
+            .offset(z: z)
+            .position(x: centerX + width / 2, y: centerY + height / 2)
+            .offset(z: spatializedElement.backOffset)
+            .opacity(opacity)
+            .hidden(!visible)
+    }
+}

package/web-spatial/view/SpatializedStatic3DView.swift

@@ -0,0 +1,72 @@
+import RealityKit
+import SwiftUI
+
+struct SpatializedStatic3DView: View {
+    @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
+    @Environment(SpatialScene.self) var spatialScene: SpatialScene
+
+    private var spatializedStatic3DElement: SpatializedStatic3DElement {
+        return spatializedElement as! SpatializedStatic3DElement
+    }
+    
+    func onLoadSuccess() {
+        spatialScene.sendWebMsg(spatializedElement.id, ModelLoadSuccess())
+    }
+
+    func onLoadFailure() {
+        spatialScene.sendWebMsg(spatializedElement.id, ModelLoadFailure())
+    }
+
+    @ViewBuilder
+    var body: some View {
+        let depth = spatializedElement.depth
+        let transform = spatializedStatic3DElement.modelTransform
+        let translation = transform.translation
+        let scale = transform.scale
+        let rotation = transform.rotation!
+        let x = translation.x
+        let y = translation.y
+        let z = translation.z
+        
+        let enableGesture = spatializedElement.enableGesture
+        if let url = URL(string: spatializedStatic3DElement.modelURL) {
+            Model3D(url: url) { newPhase in
+                switch newPhase {
+                case .empty:
+                    ProgressView()
+
+                case let .success(resolvedModel3D):
+                    resolvedModel3D
+                        .resizable(true)
+                        .aspectRatio(
+                            nil,
+                            contentMode: .fit
+                        )
+                        .if(!depth.isZero){ view in view.scaledToFit3D()}
+                        .onAppear {
+                            self.onLoadSuccess()
+                        }
+                        .if(enableGesture) { view in view.hoverEffect()}
+                case .failure:
+                    Text("").onAppear {
+                        self.onLoadFailure()
+                    }
+                @unknown default:
+                    EmptyView()
+                }
+            }
+            .scaleEffect(
+                x: scale.width,
+                y: scale.height,
+                z: scale.depth
+            )
+            .rotation3DEffect(
+                rotation
+            )
+            .offset(x: x, y: y)
+            .offset(z: z)
+        } else {
+            EmptyView()
+        }
+    }
+}

package/web-spatial/{views → view/view-modifier}/MaterialWithBorderCornerModifier.swift

@@ -49,10 +49,17 @@ struct CornerRadius: Codable {
 struct MaterialWithBorderCornerModifier: ViewModifier {
     let backgroundMaterial: BackgroundMaterial
     let cornerRadius: CornerRadius
+    let windowStyle: SpatialScene.WindowStyle
 
-    init(_ backgroundMaterial: BackgroundMaterial, _ cornerRadius: CornerRadius) {
+    init(
+        _ backgroundMaterial: BackgroundMaterial,
+        _ cornerRadius: CornerRadius,
+        _ windowStyle: SpatialScene
+            .WindowStyle
+    ) {
         self.backgroundMaterial = backgroundMaterial
         self.cornerRadius = cornerRadius
+        self.windowStyle = windowStyle
     }
 
     func body(content: Content) -> some View {
@@ -60,11 +67,20 @@ struct MaterialWithBorderCornerModifier: ViewModifier {
 
         switch backgroundMaterial {
         case .GlassMaterial:
-            content
-                .glassBackgroundEffect(
-                    in: .rect(cornerRadii: radii),
-                    displayMode: .always
-                )
+            if windowStyle == .volume {
+                content
+                    .glassBackgroundEffect(
+                        in: .rect(cornerRadii: radii),
+                        displayMode: .always
+                    )
+            } else {
+                content
+                    .glassBackgroundEffect(
+                        in: .rect(cornerRadii: radii),
+                        displayMode: .always
+                    )
+                    .frame(depth: 0)
+            }
 
         case .RegularMaterial:
             content
@@ -89,9 +105,13 @@ struct MaterialWithBorderCornerModifier: ViewModifier {
 }
 
 extension View {
-    func materialWithBorderCorner(_ backgroundMaterial: BackgroundMaterial, _ cornerRadius: CornerRadius) -> some View {
+    func materialWithBorderCorner(_ backgroundMaterial: BackgroundMaterial, _ cornerRadius: CornerRadius, _ windowStyle: SpatialScene.WindowStyle) -> some View {
         return modifier(
-            MaterialWithBorderCornerModifier(backgroundMaterial, cornerRadius)
+            MaterialWithBorderCornerModifier(
+                backgroundMaterial,
+                cornerRadius,
+                windowStyle
+            )
         )
     }
 }

package/web-spatial/webview/SpatialWebController.swift

@@ -0,0 +1,300 @@
+import SwiftUI
+@preconcurrency import WebKit
+
+class SpatialWebController: NSObject, WKNavigationDelegate, WKScriptMessageHandlerWithReply, WKUIDelegate, UIScrollViewDelegate, WKURLSchemeHandler {
+    weak var model: SpatialWebViewModel?
+    var webview: WKWebView?
+    private var isObserving = false
+    private var navigationInvoke: ((_ data: URL) -> Bool)?
+    private var openWindowInvoke: ((_ data: URL) -> WebViewElementInfo?)?
+    private var webviewStateChangeInvoke: ((_ type: SpatialWebViewState) -> Void)?
+    private var scorllUpdateInvoke: ((_ type: ScrollState, _ point: CGPoint) -> Void)?
+    private var webviewTitle: String? = nil
+    private var firstLoad = true
+    private var jsbManager = JSBManager()
+
+    override init() {
+        WKWebView.enableFileScheme() // ensure the handler is usable
+    }
+
+    deinit {}
+
+    func registerNavigationInvoke(invoke: @escaping (_ data: URL) -> Bool) {
+        navigationInvoke = invoke
+    }
+
+    func registerOpenWindowInvoke(invoke: @escaping (_ data: URL) -> WebViewElementInfo?) {
+        openWindowInvoke = invoke
+    }
+
+    func registeJSBHandler<T: CommandDataProtocol>(_ type: T.Type, _ event: @escaping (T, @escaping JSBManager.ResolveHandler<Encodable>) -> Void) {
+        jsbManager.register(type, event)
+    }
+
+    func registeJSBHandler<T: CommandDataProtocol>(_ type: T.Type, _ event: @escaping (@escaping JSBManager.ResolveHandler<Encodable>) -> Void) {
+        jsbManager.register(type, event)
+    }
+
+    func unregisterJSBHandler<T: CommandDataProtocol>(_ type: T.Type) {
+        jsbManager.remove(type)
+    }
+
+    func clearJSBHandler() {
+        jsbManager.clear()
+    }
+
+    func mockJSB(_ command: String) {
+        jsbManager.handlerMessage(command)
+    }
+
+    func registerWebviewStateChangeInvoke(invoke: @escaping (_ type: SpatialWebViewState) -> Void) {
+        webviewStateChangeInvoke = invoke
+    }
+
+    func registerScrollUpdateInvoke(invoke: @escaping (_ type: ScrollState, _ point: CGPoint) -> Void) {
+        scorllUpdateInvoke = invoke
+    }
+
+    func setWebViewTitle(_ title: String) {
+        webviewTitle = title
+        if webview != nil {
+            callJS("document.title='\(title)'")
+        }
+    }
+
+    // navigation request
+    // SpatialDiv/forcestyle/normal web link protocol
+    func webView(_ webView: WKWebView, decidePolicyFor navigationAction: WKNavigationAction, decisionHandler: @escaping (WKNavigationActionPolicy) -> Swift.Void) {
+        let deciside = navigationInvoke?(navigationAction.request.url!)
+        if deciside == true {
+            if !firstLoad{
+                webviewStateChangeInvoke?(.didUnload)
+            }
+            firstLoad = false
+        }
+        var needAllow = deciside ?? false
+        
+        if !needAllow{
+            UIApplication.shared.open(navigationAction.request.url!, options: [:], completionHandler: nil)
+        }
+        decisionHandler(needAllow ? .allow : .cancel)
+    }
+
+    // open window request
+    func webView(
+        _ webView: WKWebView,
+        createWebViewWith configuration: WKWebViewConfiguration,
+        for navigationAction: WKNavigationAction,
+        windowFeatures: WKWindowFeatures
+    ) -> WKWebView? {
+        if let modelInfo = openWindowInvoke?(navigationAction.request.url!) {
+            modelInfo.element.load(configuration, modelInfo.id)
+            return modelInfo.element.getController().webview
+        }
+        print("no webview")
+        return nil
+    }
+
+    // invoke jsb
+    func userContentController(
+        _ userContentController: WKUserContentController,
+        didReceive message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void
+    ) {
+//        let promise = JSBManager.Promise(replyHandler)
+        jsbManager.handlerMessage(message.body as! String, replyHandler)
+    }
+
+    // custom scheme request
+    func webView(_ webView: WKWebView, start urlSchemeTask: any WKURLSchemeTask) {
+        print("urlSchemeTask")
+        let url = urlSchemeTask.request.url
+        if url!.absoluteString.starts(with: "file://") {
+            let urlRequest = urlSchemeTask.request
+
+            let session = URLSession(configuration: URLSessionConfiguration.default)
+            let dataTask = session.dataTask(with: urlRequest) { [task = urlSchemeTask as AnyObject] data, response, _ in
+                guard let task = task as? WKURLSchemeTask else { return }
+
+                task.didReceive(response!)
+                task.didReceive(data!)
+                task.didFinish()
+            }
+            dataTask.resume()
+        }
+    }
+
+    func webView(_ webView: WKWebView, stop urlSchemeTask: any WKURLSchemeTask) {}
+    func webView(_ webView: WKWebView, didStartProvisionalNavigation: WKNavigation!) {
+        webviewStateChangeInvoke?(.didStartLoad)
+    }
+
+    func webView(_ webView: WKWebView, didCommit navigation: WKNavigation!) {
+        webviewStateChangeInvoke?(.didReceive)
+    }
+
+    func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
+        webviewStateChangeInvoke?(.didFinishLoad)
+        if webviewTitle != nil {
+            callJS("document.title='\(webviewTitle!)'")
+        }
+        // flush pending calljs comand
+        isPageLoaded = true
+        flushJSQueue()
+    }
+
+    func webView(_ webView: WKWebView, decidePolicyFor navigationResponse: WKNavigationResponse, decisionHandler: @escaping (WKNavigationResponsePolicy) -> Swift.Void) {
+        decisionHandler(.allow)
+    }
+
+    func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, withError error: Error) {
+        if let urlError = (error as? URLError) {
+            if urlError.code == .cannotConnectToHost {
+                webviewStateChangeInvoke?(.didFailLoad)
+            }
+        }
+    }
+
+    func webViewDidClose(_ webView: WKWebView) {
+        webviewStateChangeInvoke?(.didClose)
+    }
+
+    func webView(_ webView: WKWebView, didReceive challenge: URLAuthenticationChallenge, completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void) {
+        guard let serverTrust = challenge.protectionSpace.serverTrust else { return completionHandler(.useCredential, nil) }
+        let exceptions = SecTrustCopyExceptions(serverTrust)
+        SecTrustSetExceptions(serverTrust, exceptions)
+        completionHandler(.useCredential, URLCredential(trust: serverTrust))
+    }
+
+    func scrollViewWillBeginDragging(_ scrollView: UIScrollView) {
+        scorllUpdateInvoke?(.start, scrollView.contentOffset)
+    }
+
+    func scrollViewDidScroll(_ scrollView: UIScrollView) {
+        scorllUpdateInvoke?(.update, scrollView.contentOffset)
+    }
+
+    func scrollViewDidEndDecelerating(_ scrollView: UIScrollView) {
+        scorllUpdateInvoke?(.end, scrollView.contentOffset)
+    }
+
+    func scrollViewDidEndDragging(_ scrollView: UIScrollView, willDecelerate decelerate: Bool) {
+        if !decelerate {
+            scorllUpdateInvoke?(.end, scrollView.contentOffset)
+        } else {
+            scorllUpdateInvoke?(.release, scrollView.contentOffset)
+        }
+    }
+
+    func startObserving() {
+        guard !isObserving else { return }
+        webview?.addObserver(self, forKeyPath: #keyPath(WKWebView.url), options: .new, context: nil)
+        isObserving = true
+    }
+
+    func stopObserving() {
+        guard isObserving else { return }
+        webview?.removeObserver(self, forKeyPath: #keyPath(WKWebView.url))
+        isObserving = false
+    }
+
+    override func observeValue(
+        forKeyPath keyPath: String?,
+        of object: Any?,
+        change: [NSKeyValueChangeKey: Any]?,
+        context: UnsafeMutableRawPointer?
+    ) {
+        if keyPath == #keyPath(WKWebView.url),
+           let url = (object as? WKWebView)?.url?.absoluteString
+        {
+            DispatchQueue.main.async {
+//                print("url change", url)
+                self.model?.url = url
+            }
+        }
+    }
+
+    func destroy() {
+        destroyView()
+        navigationInvoke = nil
+        openWindowInvoke = nil
+        webviewStateChangeInvoke = nil
+        scorllUpdateInvoke = nil
+        model = nil
+    }
+    
+    private var state:SpatialWebViewState?
+
+    func destroyView() {
+        stopObserving()
+        if webview != nil {
+            webview?.stopLoading()
+            webview?.configuration.userContentController.removeScriptMessageHandler(forName: "bridge")
+            webview?.uiDelegate = nil
+            webview?.navigationDelegate = nil
+            webview?.scrollView.delegate = nil
+            webview = nil
+            webviewStateChangeInvoke?(.didDestroyView)
+        }
+    }
+    
+    private var isPageLoaded = false
+    
+    private var jsQueue: [String] = []
+    
+    private func enqueueJS(_ js: String) {
+        jsQueue.append(js)
+    }
+    
+    private func flushJSQueue() {
+        guard !jsQueue.isEmpty else { return }
+        let combined = jsQueue.joined(separator: ";")
+        callJS(combined)
+        jsQueue.removeAll()
+    }
+
+    func callJS(_ js: String) {
+        if webview != nil && isPageLoaded {
+            webview!.evaluateJavaScript(js)
+        } else {
+            enqueueJS(js)
+        }
+    }
+}
+
+enum ScrollState {
+    case start
+    case update
+    case release
+    case end
+}
+
+// extend webview to support file://
+ @available(iOS 11.0, *)
+ extension WKWebView {
+    /// WKWebView,  Support setting file scheme in configuration
+    public private(set) static var isEnableFileSupport = false
+    public static func enableFileScheme() {
+        /// This method supports adapting supported files through Configuration, but cannot be cancelled (Configuration is immutable).
+        if !isEnableFileSupport {
+            switchHandlesURLScheme()
+        }
+    }
+
+    private static func switchHandlesURLScheme() {
+        if
+            case let cls = WKWebView.self,
+            let m1 = class_getClassMethod(cls, NSSelectorFromString("handlesURLScheme:")),
+            let m2 = class_getClassMethod(cls, #selector(WKWebView.wrapHandles(urlScheme:)))
+        {
+            method_exchangeImplementations(m1, m2)
+            isEnableFileSupport = !isEnableFileSupport
+        }
+    }
+
+    /// Return true if WKWebview supports handling this protocol, but WKWebview supports HTTP by default, so return false to support using custom HTTP Handler
+    @objc private dynamic
+    static func wrapHandles(urlScheme: String) -> Bool {
+        if urlScheme == "file" { return false }
+        return wrapHandles(urlScheme: urlScheme)
+    }
+ }

package/web-spatial/webview/SpatialWebView.swift

@@ -0,0 +1,34 @@
+import SwiftUI
+@preconcurrency import WebKit
+
+struct SpatialWebView: UIViewRepresentable {
+    weak var model: SpatialWebViewModel? = nil
+    var url: URL = .init(filePath: "/")
+    private var webviewStateChangeInvoke: ((_ type: SpatialWebViewState) -> Void)?
+
+    func makeUIView(context: Context) -> WKWebView {
+        webviewStateChangeInvoke?(.didMakeView)
+        return model!.getController().webview!
+    }
+
+    func makeCoordinator() -> SpatialWebController {
+        return model!.getController()
+    }
+
+    func updateUIView(_ webView: WKWebView, context: Context) {
+        webviewStateChangeInvoke?(.didUpdateView)
+    }
+
+    mutating func registerWebviewStateChangeInvoke(invoke: @escaping (_ type: SpatialWebViewState) -> Void) {
+        webviewStateChangeInvoke = invoke
+    }
+
+    mutating func destroy() {
+        webviewStateChangeInvoke = nil
+        model = nil
+    }
+
+    static func dismantleUIView(_ uiView: WKWebView, coordinator: SpatialWebController) {
+//        print("dismantleUIView", coordinator.model?.id)
+    }
+}