@webspatial/platform-visionos 1.0.4 → 1.0.5

package/web-spatial/view/SpatialSceneContentView.swift

@@ -0,0 +1,218 @@
+import RealityKit
+import SwiftUI
+
+struct SpatialSceneContentView: View {
+    @State var sceneId: String
+    var width: Double
+    var height: Double
+
+    var body: some View {
+        if let spatialScene = SpatialApp.Instance.getScene(sceneId) {
+            ZStack(alignment: Alignment.topLeading) {
+                // Display the main webview
+                if spatialScene.didFailLoad {
+                    VStack {
+                        Text("Failed to load webpage. Is the server running?")
+                            .foregroundColor(.white)
+                        Button("Reload") {
+                            let url = spatialScene.spatialWebViewModel.url
+
+                            spatialScene.spatialWebViewModel.load(url)
+                        }
+                        .foregroundColor(.white)
+                    }
+                    .frame(maxWidth: .infinity, maxHeight: .infinity).glassBackgroundEffect()
+                } else {
+                    spatialScene.getView()
+                        .materialWithBorderCorner(
+                            spatialScene.backgroundMaterial,
+                            spatialScene.cornerRadius,
+                            spatialScene.windowStyle
+                        )
+                        .frame(width: width, height: height)
+                        .offset(z: -1)
+
+                    let childrenOfSpatialized2DElement: [SpatializedElement] = Array(spatialScene.getChildrenOfType(.Spatialized2DElement).values)
+
+                    ForEach(childrenOfSpatialized2DElement, id: \.id) { child in
+                        SpatializedElementView(parentScrollOffset: spatialScene.scrollOffset) {
+                            Spatialized2DElementView()
+                        }
+                        .environment(child)
+                    }
+
+                    let childrenOfSpatializedStatic3DElement: [SpatializedElement] = Array(spatialScene.getChildrenOfType(.SpatializedStatic3DElement).values)
+
+                    ForEach(childrenOfSpatializedStatic3DElement, id: \.id) { child in
+                        SpatializedElementView(parentScrollOffset: spatialScene.scrollOffset) {
+                            SpatializedStatic3DView()
+                        }
+                        .environment(child)
+                    }
+
+                    let childrenOfSpatializedDynamic3DElement: [SpatializedElement] = Array(spatialScene.getChildrenOfType(.SpatializedDynamic3DElement).values)
+
+                    ForEach(childrenOfSpatializedDynamic3DElement, id: \.id) { child in
+                        SpatializedElementView(parentScrollOffset: spatialScene.scrollOffset) {
+                            SpatializedDynamic3DView()
+                        }
+                        .environment(child)
+                    }
+                }
+            }
+            .opacity(spatialScene.opacity)
+            .environment(spatialScene)
+            .coordinateSpace(name: "SpatialScene")
+        }
+    }
+}
+
+struct PreviewSpatializedStatic3DElement: View {
+    var sceneId: String
+
+    init() {
+        let spatialScene = SpatialApp.Instance.createScene(
+            "http://localhost:5173/",
+            .window,
+            .visible
+        )
+
+        let spatializedStatic3DElement: SpatializedStatic3DElement = spatialScene.createSpatializedElement(
+            .SpatializedStatic3DElement
+        )
+        spatializedStatic3DElement.transform.translation.x = 200
+        spatializedStatic3DElement.transform.translation.y = 300
+        spatializedStatic3DElement.transform.translation.z = 0
+        spatializedStatic3DElement.name = "jack"
+
+        spatializedStatic3DElement.width = 200
+        spatializedStatic3DElement.height = 100
+
+        spatializedStatic3DElement.modelURL = "http://localhost:5173/public/modelasset/cone.usdz"
+        spatializedStatic3DElement.setParent(spatialScene)
+
+        let spatializedStatic3DElementB: SpatializedStatic3DElement = spatialScene.createSpatializedElement(
+            .SpatializedStatic3DElement
+        )
+        spatializedStatic3DElementB.transform.translation.x = 700
+        spatializedStatic3DElementB.transform.translation.y = 300
+        spatializedStatic3DElementB.transform.translation.z = 0
+        spatializedStatic3DElementB.width = 200
+        spatializedStatic3DElementB.height = 200
+        spatializedStatic3DElementB.name = "tom"
+
+        spatializedStatic3DElementB.modelURL = "http://localhost:5173/public/modelasset/vehicle-speedster.usdz"
+        spatializedStatic3DElementB.setParent(spatialScene)
+
+        sceneId = spatialScene.id
+
+        print("spatialScene \(spatialScene)")
+
+        let spatializedElement: SpatializedDynamic3DElement = spatialScene.createSpatializedElement(
+            .SpatializedDynamic3DElement
+        )
+
+        spatializedElement.name = "dynamicCubes"
+        spatializedElement.transform.translation.x = 400
+        spatializedElement.transform.translation.y = 400
+        spatializedElement.transform.translation.z = 0
+        spatializedElement.width = 200
+        spatializedElement.height = 200
+        spatializedElement.setParent(spatialScene)
+    }
+
+    var body: some View {
+        SpatialSceneContentView(sceneId: sceneId, width: 1200, height: 1000)
+    }
+}
+
+struct PreviewSpatialized2DElement: View {
+    var sceneId: String
+
+    init() {
+        let spatialScene = SpatialApp.Instance.createScene(
+            "http://localhost:5173/",
+            .window,
+            .visible
+        )
+
+        let spatializedElementA: Spatialized2DElement = spatialScene.createSpatializedElement(
+            .Spatialized2DElement
+        )
+        spatializedElementA.transform.translation.x = 200
+        spatializedElementA.transform.translation.y = 300
+        spatializedElementA.transform.translation.z = 0
+        spatializedElementA.name = "jack"
+
+        spatializedElementA.width = 200
+        spatializedElementA.height = 100
+        spatializedElementA.load("http://localhost:5173/src/")
+
+        spatializedElementA.setParent(spatialScene)
+
+        let spatializedElementB: Spatialized2DElement = spatialScene.createSpatializedElement(
+            .Spatialized2DElement
+        )
+        spatializedElementB.transform.translation.x = 400
+        spatializedElementB.transform.translation.y = 300
+        spatializedElementB.transform.translation.z = 0
+        spatializedElementB.name = "jack"
+
+        spatializedElementB.width = 200
+        spatializedElementB.height = 100
+        spatializedElementB.load("http://localhost:5173/src/embed/")
+        spatializedElementB.setParent(spatialScene)
+
+        sceneId = spatialScene.id
+
+        print("spatialScene \(spatialScene)")
+    }
+
+    var body: some View {
+        SpatialSceneContentView(sceneId: sceneId, width: 1200, height: 1000)
+    }
+}
+
+struct PreviewSpatializedDynamic3DElement: View {
+    var sceneId: String
+
+    init() {
+        let spatialScene = SpatialApp.Instance.createScene(
+            "http://localhost:5173/",
+            .window,
+            .visible
+        )
+
+        let spatializedElement: SpatializedDynamic3DElement = spatialScene.createSpatializedElement(
+            .SpatializedDynamic3DElement
+        )
+
+        spatializedElement.name = "dynamicCubes"
+        spatializedElement.transform.translation.x = 100
+        spatializedElement.transform.translation.y = 100
+        spatializedElement.transform.translation.z = 0
+        spatializedElement.width = 200
+        spatializedElement.height = 200
+        spatializedElement.setParent(spatialScene)
+
+        sceneId = spatialScene.id
+
+        print("spatialScene \(spatialScene)")
+    }
+
+    var body: some View {
+        SpatialSceneContentView(sceneId: sceneId, width: 1200, height: 1000)
+    }
+}
+
+#Preview("PreviewSpatializedDynamic3DElement") {
+    PreviewSpatializedDynamic3DElement()
+}
+
+#Preview("PreviewSpatializedStatic3DElementWithRotation") {
+    PreviewSpatializedStatic3DElement()
+}
+
+#Preview("PreviewSpatialized2DElement") {
+    PreviewSpatialized2DElement()
+}

package/web-spatial/view/SpatialSceneView.swift

@@ -0,0 +1,53 @@
+import _RealityKit_SwiftUI
+import SwiftUI
+
+struct SpatialSceneView: View {
+    @State var spatialScene: SpatialScene
+    @State private var windowResizeInProgress = false
+    @State private var timer: Timer?
+
+    var body: some View {
+        GeometryReader3D { proxy3D in
+            let width = proxy3D.size.width
+            let height = proxy3D.size.height
+
+            SceneHandlerUIView(spatialScene: spatialScene).onChange(of: proxy3D.size) {
+                windowResizeInProgress = true
+                if timer != nil {
+                    timer!.invalidate()
+                }
+                // If we don't detect resolution change after x seconds we treat the resize as complete
+                timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: false) { _ in
+                    windowResizeInProgress = false
+                    timer = nil
+                }
+                spatialScene.updateSize3D(proxy3D.size)
+            }
+            .onAppear {
+                spatialScene.moveToState(.visible, nil)
+                spatialScene.updateSize3D(proxy3D.size)
+            }
+
+            if windowResizeInProgress {
+                let x = width / 2
+                let y = height / 2
+                VStack {}.frame(width: width, height: height).glassBackgroundEffect().padding3D(.front, -100_000)
+                    .position(x: x, y: y)
+            } else {
+                SpatialSceneContentView(sceneId: spatialScene.id, width: width, height: height)
+                    .ornament(attachmentAnchor: .scene(.top), contentAlignment: .center) {
+                        if pwaManager.display != .fullscreen {
+                            ZStack {
+                                SpatialNavView(
+                                    spatialScene: spatialScene
+                                )
+                                .offset(y: -15)
+                            }.frame(height: 100)
+                        }
+                    }.volumeBaseplateVisibility(
+                        spatialScene.sceneConfig?.baseplateVisibility ?? .automatic
+                    )
+            }
+        }
+    }
+}

package/web-spatial/view/Spatialized2DElementView.swift

@@ -0,0 +1,96 @@
+import RealityKit
+import SwiftUI
+
+extension View {
+    @ViewBuilder
+    func `if`<Content: View>(_ condition: Bool, transform: (Self) -> Content) -> some View {
+        if condition {
+            transform(self)
+        } else {
+            self
+        }
+    }
+}
+
+class Spatialized2DViewGestureData {
+    var dragStarted = false
+    var dragStart: CGFloat = 0.0
+    var dragVelocity: CGFloat = 0.0
+}
+
+struct Spatialized2DElementView: View {
+    @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
+    @Environment(SpatialScene.self) var spatialScene: SpatialScene
+
+    private var spatialized2DElement: Spatialized2DElement {
+        return spatializedElement as! Spatialized2DElement
+    }
+
+    @State private var gestureData = Spatialized2DViewGestureData()
+
+    var body: some View {
+        // Display child spatialized2DElements
+        ZStack(alignment: Alignment.topLeading) {
+            // Display the main webview
+            spatialized2DElement.getView()
+                .materialWithBorderCorner(
+                    spatialized2DElement.backgroundMaterial,
+                    spatialized2DElement.cornerRadius,
+                    .window
+                )
+                .simultaneousGesture(spatialized2DElement.scrollPageEnabled ? dragWebGesture : nil)
+
+            let childrenOfSpatialized2DElement: [SpatializedElement] = Array(spatialized2DElement.getChildrenOfType(.Spatialized2DElement).values)
+
+            ForEach(childrenOfSpatialized2DElement, id: \.id) { child in
+                SpatializedElementView(parentScrollOffset: spatialized2DElement.scrollOffset) {
+                    Spatialized2DElementView()
+                }
+                .environment(child)
+            }
+
+            let childrenOfSpatializedStatic3DElement: [SpatializedElement] = Array(spatialized2DElement.getChildrenOfType(.SpatializedStatic3DElement).values)
+            ForEach(childrenOfSpatializedStatic3DElement, id: \.id) { child in
+                SpatializedElementView(parentScrollOffset: spatialized2DElement.scrollOffset) {
+                    SpatializedStatic3DView()
+                }
+                .environment(child)
+            }
+
+            let childrenOfSpatializedDynamic3DElement: [SpatializedElement] = Array(spatialized2DElement.getChildrenOfType(.SpatializedDynamic3DElement).values)
+
+            ForEach(childrenOfSpatializedDynamic3DElement, id: \.id) { child in
+                SpatializedElementView(parentScrollOffset: spatialized2DElement.scrollOffset) {
+                    SpatializedDynamic3DView()
+                }
+                .environment(child)
+            }
+        }
+    }
+
+    private var dragWebGesture: some Gesture {
+        DragGesture()
+            .onChanged { gesture in
+                print("\(spatialized2DElement.name) dragWebGesture")
+                if spatialized2DElement.scrollPageEnabled {
+                    if !gestureData.dragStarted {
+                        gestureData.dragStarted = true
+                        gestureData.dragStart = (gesture.translation.height)
+                    }
+
+                    // TODO: this should have velocity
+                    let delta = gestureData.dragStart - gesture.translation.height
+                    gestureData.dragStart = gesture.translation.height
+                    spatialScene.updateDeltaScrollOffset(Vec2(x: 0, y: delta))
+                }
+            }
+            .onEnded { _ in
+                print("\(spatialized2DElement.name) dragWebGestureEnd")
+                if spatialized2DElement.scrollPageEnabled {
+                    gestureData.dragStarted = false
+                    gestureData.dragStart = 0
+                    spatialScene.stopScrolling()
+                }
+            }
+    }
+}

package/web-spatial/view/SpatializedDynamic3DView.swift

@@ -0,0 +1,104 @@
+import RealityKit
+import SwiftUI
+
+struct SpatializedDynamic3DView: View {
+    @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
+    @Environment(SpatialScene.self) var spatialScene: SpatialScene
+    @State private var isDrag = false
+    @State private var isRotate = false
+    @State private var isScale = false
+    
+    private var spatializedDynamic3DElement: SpatializedDynamic3DElement {
+        return spatializedElement as! SpatializedDynamic3DElement
+    }
+    
+    var spatialTapEvent: some Gesture{
+        SpatialTapGesture(count: 1).targetedToAnyEntity()
+            .onEnded{ value in
+                if let entity = value.entity as? SpatialEntity{
+                    spatialScene.sendWebMsg(entity.spatialId, WebSpatialTapGuestureEvent(detail: WebSpatialTapGuestureEventDetail(location3D: value.location3D)))
+                }
+                else {
+                    if let spatialEntity = SpatialEntity.findNearestParent(entity: value.entity){
+                        spatialScene.sendWebMsg(spatialEntity.spatialId, WebSpatialTapGuestureEvent(detail: WebSpatialTapGuestureEventDetail(location3D: value.location3D)))
+                    }
+                }
+        }
+    }
+    
+    var rotate3dEvent: some Gesture{
+        RotateGesture3D().targetedToAnyEntity().onChanged{ value in
+            print(value.gestureValue)
+            if let entity = value.entity as? SpatialEntity{
+                if entity.enableTap == true {
+//                    if(isRotate == false){
+//                        print("start rotate")
+//                    }
+//                    else{
+//                        print("rotating")
+//                    }
+//                    isRotate = true
+                    return
+                }
+            }
+            isRotate = false
+        }.onEnded{ value in
+            print(value.rotation)
+            print("rotate end")
+//            isRotate = false
+        }
+    }
+    
+    var magnifyEvent: some Gesture{
+        MagnifyGesture().targetedToAnyEntity().onChanged{ value in
+            print(value)
+            if let entity = value.entity as? SpatialEntity{
+                if entity.enableTap == true {
+//                    if(isScale == false){
+//                        print("start scale")
+//                    }
+//                    else{
+//                        print("scaling")
+//                    }
+//                    isScale = true
+                    return
+                }
+            }
+        }.onEnded{ value in
+            print("scale end")
+//            isScale = false
+        }
+    }
+    
+    var dragEvent: some Gesture{
+        DragGesture().targetedToAnyEntity().onChanged{ value in
+            if let entity = value.entity as? SpatialEntity{
+                if entity.enableTap == true {
+//                    if(isDrag == false){
+//                        print("start drag")
+//                    }
+//                    else{
+//                        print("dragging")
+//                    }
+//                    isDrag = true
+                    return
+                }
+            }
+            
+        }.onEnded{ value in
+            print("drag end")
+//            isDrag = false
+        }
+    }
+    
+    var body: some View {
+        RealityView(make: { content in
+            let rootEntity = spatializedDynamic3DElement.getRoot()
+            content.add(rootEntity)
+        })
+        .simultaneousGesture(spatialTapEvent)
+        .simultaneousGesture(rotate3dEvent)
+        .simultaneousGesture(dragEvent)
+        .simultaneousGesture(magnifyEvent)
+    }
+}

package/web-spatial/view/SpatializedElementView.swift

@@ -0,0 +1,178 @@
+import SwiftUI
+
+// zIndex() have some bug, so use zOrderBias to simulate zIndex effect
+let zOrderBias = 0.001
+
+struct SpatializedElementView<Content: View>: View {
+    @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
+    @Environment(SpatialScene.self) var spatialScene: SpatialScene
+
+    var parentScrollOffset: Vec2
+    var content: Content
+
+    init(parentScrollOffset: Vec2, @ViewBuilder content: () -> Content) {
+        self.parentScrollOffset = parentScrollOffset
+        self.content = content()
+    }
+
+    // Begin Interaction
+    var gesture: some Gesture {
+        DragGesture()
+            .onChanged(onDragging)
+            .onEnded(onDraggingEnded)
+            .simultaneously(with:
+                RotateGesture3D()
+                    .onChanged(onRotateGesture3D)
+                    .onEnded(onRotateGesture3DEnd)
+            )
+            .simultaneously(with:
+                MagnifyGesture()
+                    .onChanged(onMagnifyGesture)
+                    .onEnded(onMagnifyGestureEnd)
+            )
+            .simultaneously(with:
+                SpatialTapGesture(count: 1)
+                    .onEnded(onTapEnded)
+            )
+    }
+
+    private func onRotateGesture3D(_ event: RotateGesture3D.Value) {
+        if spatializedElement.enableRotateGesture || spatializedElement.enableRotateStartGesture {
+            let gestureEvent = WebSpatialRotateGuestureEvent(
+                detail: .init(
+                    rotation: event.rotation,
+                    startAnchor3D: event.startAnchor3D,
+                    startLocation3D: event.startLocation3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onRotateGesture3DEnd(_ event: RotateGesture3D.Value) {
+        if spatializedElement.enableRotateEndGesture {
+            let gestureEvent = WebSpatialRotateEndGuestureEvent(
+                detail: .init(
+                    rotation: event.rotation,
+                    startAnchor3D: event.startAnchor3D,
+                    startLocation3D: event.startLocation3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onDragging(_ event: DragGesture.Value) {
+        if spatializedElement.enableDragStartGesture || spatializedElement.enableDragGesture {
+            let gestureEvent = WebSpatialDragGuestureEvent(detail: .init(
+                location3D: event.location3D,
+                startLocation3D: event.startLocation3D,
+                translation3D: event.translation3D,
+                predictedEndTranslation3D: event.predictedEndTranslation3D,
+                predictedEndLocation3D: event.predictedEndLocation3D,
+                velocity: event.velocity
+            ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onDraggingEnded(_ event: DragGesture.Value) {
+        if spatializedElement.enableDragEndGesture {
+            let gestureEvent = WebSpatialDragEndGuestureEvent(
+                detail: .init(
+                    location3D: event.location3D,
+                    startLocation3D: event.startLocation3D,
+                    translation3D: event.translation3D,
+                    predictedEndTranslation3D: event.predictedEndTranslation3D,
+                    predictedEndLocation3D: event.predictedEndLocation3D,
+                    velocity: event.velocity
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onTapEnded(_ event: SpatialTapGesture.Value) {
+        if spatializedElement.enableTapGesture {
+            spatialScene.sendWebMsg(spatializedElement.id, WebSpatialTapGuestureEvent(detail: .init(location3D: event.location3D)))
+        }
+    }
+
+    private func onMagnifyGesture(_ event: MagnifyGesture.Value) {
+        if spatializedElement.enableMagnifyGesture || spatializedElement.enableMagnifyStartGesture {
+            let gestureEvent = WebSpatialMagnifyGuestureEvent(
+                detail: .init(
+                    magnification: event.magnification,
+                    velocity: event.velocity,
+                    startLocation3D: event.startLocation3D,
+                    startAnchor3D: event.startAnchor3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    private func onMagnifyGestureEnd(_ event: MagnifyGesture.Value) {
+        if spatializedElement.enableMagnifyEndGesture {
+            let gestureEvent = WebSpatialMagnifyEndGuestureEvent(
+                detail: .init(
+                    magnification: event.magnification,
+                    velocity: event.velocity,
+                    startLocation3D: event.startLocation3D,
+                    startAnchor3D: event.startAnchor3D
+                ))
+            spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
+        }
+    }
+
+    // End Interaction
+
+    @ViewBuilder
+    var body: some View {
+        let transform = spatializedElement.transform
+        let translation = transform.translation
+        let scale = transform.scale
+        let rotation = transform.rotation!
+
+        let width = spatializedElement.width
+        let height = spatializedElement.height
+        let depth = spatializedElement.depth
+        let anchor = spatializedElement.rotationAnchor
+
+        let centerX = spatializedElement.clientX - (spatializedElement.scrollWithParent ? parentScrollOffset.x : 0)
+        let centerY = spatializedElement.clientY - (spatializedElement.scrollWithParent ? parentScrollOffset.y : 0)
+
+        let opacity = spatializedElement.opacity
+        let visible = spatializedElement.visible
+        let enableGesture = spatializedElement.enableGesture
+
+        let z = translation.z + (spatializedElement.zIndex * zOrderBias)
+        let smallOffset = z == 0.0 ? 0.0001 : 0
+
+        content.simultaneousGesture(enableGesture ? gesture : nil)
+            .frame(width: width, height: height)
+            .frame(depth: depth, alignment: .back)
+            .onGeometryChange3D(for: AffineTransform3D.self) { proxy in
+                let rect3d = proxy.frame(in: .named("SpatialScene"))
+                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerClientCube(origin: rect3d.origin, size: rect3d.size))
+                return proxy.transform(in: .named("SpatialScene"))!
+            } action: { _, new in
+                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerTransform(detail: new))
+            }
+            .frame(depth: 0, alignment: .back)
+            // use .offset(smallVal) to workaround for glassEffect not working and small width/height spatialDiv not working
+            .offset(z: smallOffset)
+            .scaleEffect(
+                x: scale.width,
+                y: scale.height,
+                z: scale.depth,
+                anchor: anchor
+            )
+            .rotation3DEffect(
+                rotation,
+                anchor: anchor
+            )
+            .offset(x: translation.x, y: translation.y)
+            .offset(z: z)
+            .position(x: centerX + width / 2, y: centerY + height / 2)
+            .offset(z: spatializedElement.backOffset)
+            .opacity(opacity)
+            .hidden(!visible)
+    }
+}