@webspatial/builder 0.0.1 → 0.0.3

package/template/visionOSApp/web-spatial/views/SpatialViewUI.swift

@@ -0,0 +1,168 @@
+//
+//  SpatialViewUI.swift
+//  web-spatial
+//
+//  Created by ByteDance on 11/6/24.
+//
+import RealityKit
+import SwiftUI
+
+extension View {
+    @ViewBuilder
+    func `if`<Content: View>(_ condition: Bool, transform: (Self) -> Content) -> some View {
+        if condition {
+            transform(self)
+        } else {
+            self
+        }
+    }
+}
+
+struct SpatialViewUI: View {
+    @Environment(SpatialEntity.self) var ent: SpatialEntity
+    @State var isRoot = false
+
+    // Entity which will contain all the content of this realityView and scale to fit frame
+    @State var world = Entity()
+    @State var portal = Entity()
+    @State var light = PointLight()
+    @State var portalModel = ModelComponent(
+        mesh: .generatePlane(width: 1.0, height: 1.0, cornerRadius: 0.0),
+        materials: [PortalMaterial()]
+    )
+    @State var worldComponent = WorldComponent()
+
+    private func toJson(val: SIMD3<Float>) -> String {
+        return "{x: " + String(val.x) + ",y: " + String(val.y) + ",z: " + String(val.z) + "}"
+    }
+
+    var dragGesture: some Gesture {
+        DragGesture(minimumDistance: 0).handActivationBehavior(.automatic)
+            .targetedToAnyEntity()
+            .onChanged { value in
+                let startPos = value.convert(value.startLocation3D, from: .local, to: .scene)
+                let translate = value.convert(value.location3D, from: .local, to: .scene)
+                let spatialEntity = value.entity.components[SpatialBridgeComponent.self]!.spatialEntity
+                let ic = spatialEntity.getComponent(SpatialInputComponent.self)!
+
+                if !ic.isDragging {
+                    ic.isDragging = true
+                    ic.trackedPosition = startPos
+                    let delta = translate - ic.trackedPosition
+                    ic.trackedPosition = translate
+
+                    ic.wv!.fireComponentEvent(componentId: ic.id, data: "{eventType: 'dragstart', translate: " + toJson(val: delta) + "}")
+                } else {
+                    let delta = translate - ic.trackedPosition
+                    ic.trackedPosition = translate
+                    ic.wv!.fireComponentEvent(componentId: ic.id, data: "{eventType: 'drag', translate: " + toJson(val: delta) + "}")
+                }
+            }
+            .onEnded { value in
+                let spatialEntity = value.entity.components[SpatialBridgeComponent.self]!.spatialEntity
+                let ic = spatialEntity.getComponent(SpatialInputComponent.self)!
+                ic.wv!.fireComponentEvent(componentId: ic.id, data: "{eventType: 'dragend'}")
+                ic.isDragging = false
+            }
+    }
+
+    var body: some View {
+        if let viewComponent = ent.getComponent(SpatialViewComponent.self) {
+            GeometryReader3D { proxy in
+                // Get dimensions of the frame
+                let proxySize3d = proxy.frame(in: .local)
+
+                RealityView { _, _ in
+                } update: { content, attachments in
+                    // Scale content so it will be a 1x1x1 space and not exceed the frame
+                    let viewSpaceDimensions = content.convert(proxySize3d, from: .local, to: content)
+                    let newScale = min(viewSpaceDimensions.extents.x, viewSpaceDimensions.extents.y)
+
+                    world.transform.scale.x = newScale
+                    world.transform.scale.y = newScale
+                    world.transform.scale.z = newScale
+                    portal.transform.scale.x = newScale
+                    portal.transform.scale.y = newScale
+                    portal.transform.scale.z = newScale
+
+                    if !isRoot {
+                        // Pull out content so volume sits in front of the page
+                        world.transform.translation.z = world.transform.scale.z / 2
+                    }
+
+                    for (_, entity) in ent.getEntities() {
+                        world.addChild(entity.modelEntity)
+                    }
+
+                    // Add attachments for window entities
+                    let entities = ent.getEntities().filter { _, entity in
+                        entity.coordinateSpace == .APP && entity.hasComponent(SpatialWindowComponent.self)
+                    }
+                    for key in Array(entities.keys) {
+                        let e = entities[key]!
+                        let windowComponent = e.getComponent(SpatialWindowComponent.self)
+                        if windowComponent != nil && e.coordinateSpace == .APP {
+                            if let windowAttachment = attachments.entity(for: key) {
+                                if e.modelEntity.children.count == 0 {
+                                    e.modelEntity.addChild(windowAttachment, preservingWorldTransform: false)
+
+                                    // Scale the window to fit the resolution to unit ratio as defined by setResolution API
+                                    let b = windowAttachment.attachment.bounds
+                                    let wv = e.getComponent(SpatialWindowComponent.self)!
+                                    let scaleFact = (Float(wv.resolutionX) / 1360.0) / (b.max.x - b.min.x)
+                                    windowAttachment.scale.x = scaleFact
+                                    windowAttachment.scale.y = scaleFact
+                                    windowAttachment.scale.z = scaleFact
+                                }
+                            }
+                        }
+                    }
+                    if viewComponent.isPortal {
+                        // Setup portal
+                        portal.components.set(portalModel)
+                        portal.transform.translation.z = 0.0001 // avoid z fighting
+                        if !portal.components.has(PortalComponent.self) {
+                            portal.components.set(PortalComponent(target: world))
+                        }
+
+                        // Setup default light
+                        light.light.intensity = 5000
+                        light.position.z = 2
+                        light.position.y = 1
+                        light.position.x = 0.5
+                        world.addChild(light)
+
+                        // Position volume behind portal instead of in front
+                        world.transform.translation.z *= -1
+
+                        // Add portal to scene
+                        world.components.set(worldComponent)
+                        content.add(portal)
+                    } else {
+                        // Remove portal elements/components
+                        content.remove(portal)
+                        world.components.remove(WorldComponent.self)
+                        world.removeChild(light)
+                    }
+
+                    content.add(world)
+                }
+                attachments: {
+                    // Create an attachment for each window component
+                    let entities = ent.getEntities().filter { _, entity in
+                        entity.coordinateSpace == .APP && entity.hasComponent(SpatialWindowComponent.self)
+                    }
+                    ForEach(Array(entities.keys), id: \.self) { key in
+                        let entity = entities[key]!
+                        let wv = entity.getComponent(SpatialWindowComponent.self)!
+                        Attachment(id: key) {
+                            SpatialWebViewUI().environment(entity).frame(width: wv.resolutionX, height: wv.resolutionY)
+                        }
+                    }
+                }.gesture(dragGesture).if(!isRoot) { view in
+                    view.clipped()
+                }
+            }
+        }
+    }
+}

package/template/visionOSApp/web-spatial/views/SpatialWebViewUI.swift

@@ -0,0 +1,187 @@
+//
+//  SpatialWebViewUI.swift
+//  web-spatial
+//
+//  Created by ByteDance on 5/9/24.
+//
+
+import RealityKit
+import SwiftUI
+
+// Using scrollview has some side effects so only use it on elements we want to clip the edges of
+// Seems only scrollview has this clipping property so far on visionOS otherwise we would use ZStack
+struct OptionalClip<Content: View>: View {
+    var clipEnabled = true
+    let viewBuilder: () -> Content
+
+    var body: some View {
+        if clipEnabled {
+            ScrollView {
+                viewBuilder()
+            }.offset(z: CGFloat(0)).frame(maxWidth: .infinity, maxHeight: .infinity).scrollDisabled(true)
+        } else {
+            viewBuilder()
+        }
+    }
+}
+
+struct SpatialWebViewUI: View {
+    @Environment(SpatialEntity.self) var ent: SpatialEntity
+    var body: some View {
+        if let wv = ent.getComponent(SpatialWindowComponent.self) {
+            let parentYOffset = Float(wv.scrollOffset.y)
+
+            let childEntities = ent.getEntities()
+
+            // Display child entities of the webview
+            VStack(alignment: .trailing, spacing: 20) {
+                if wv.isRootWebview() {
+                    NavView(swc: wv)
+                }
+
+                ZStack {
+                    OptionalClip(clipEnabled: ent.coordinateSpace != .ROOT && wv.isScrollEnabled()) {
+                        ZStack {
+                            ForEach(Array(childEntities.keys), id: \.self) { key in
+                                if let e = childEntities[key] {
+                                    let _ = e.forceUpdate ? 0 : 0
+                                    if let childWindowcomponent = e.getComponent(SpatialWindowComponent.self) {
+                                        if e.coordinateSpace == .DOM {
+                                            let view = childWindowcomponent
+                                            let x = CGFloat(e.modelEntity.position.x)
+                                            let y = CGFloat(e.modelEntity.position.y - (view.scrollWithParent ? parentYOffset : 0))
+                                            let z = CGFloat(e.modelEntity.position.z)
+                                            let width = CGFloat(view.resolutionX)
+                                            let height = CGFloat(view.resolutionY)
+                                            let anchor = view.rotationAnchor
+
+                                            // Matrix = MTranslate X MRotate X MScale
+                                            SpatialWebViewUI().environment(e)
+                                                .frame(width: width, height: height)
+                                                // use .offset(smallVal) to workaround for glassEffect not working and small width/height spatialDiv not working
+                                                .offset(z: 0.0001)
+                                                .scaleEffect(
+                                                    x: CGFloat(e.modelEntity.scale.x),
+                                                    y: CGFloat(e.modelEntity.scale.y),
+                                                    z: CGFloat(e.modelEntity.scale.z),
+                                                    anchor: anchor
+                                                )
+                                                .rotation3DEffect(
+                                                    Rotation3D(simd_quatf(
+                                                        ix: e.modelEntity.orientation.vector.x,
+                                                        iy: e.modelEntity.orientation.vector.y,
+                                                        iz: e.modelEntity.orientation.vector.z,
+                                                        r: e.modelEntity.orientation.vector.w
+                                                    )),
+                                                    anchor: anchor
+                                                )
+
+                                                .position(x: x, y: y)
+                                                .offset(z: z)
+                                                .zIndex(e.zIndex)
+                                                .gesture(
+                                                    DragGesture()
+                                                        .onChanged { gesture in
+                                                            let scrollEnabled = view.isScrollEnabled()
+                                                            if !scrollEnabled, wv.isScrollEnabled() {
+                                                                if !view.dragStarted {
+                                                                    view.dragStarted = true
+                                                                    view.dragStart = (gesture.translation.height)
+                                                                }
+
+                                                                // TODO: this should have velocity
+                                                                let delta = view.dragStart - gesture.translation.height
+                                                                view.dragStart = gesture.translation.height
+                                                                wv.updateScrollOffset(delta: delta)
+                                                            }
+                                                        }
+                                                        .onEnded { _ in
+                                                            let scrollEnabled = view.isScrollEnabled()
+                                                            if !scrollEnabled, wv.isScrollEnabled() {
+                                                                view.dragStarted = false
+                                                                view.dragStart = 0
+
+                                                                wv.stopScrolling()
+                                                            }
+                                                        }
+                                                )
+                                        }
+                                    }
+                                }
+                            }
+
+                            // Model3D content
+                            ForEach(Array(childEntities.keys), id: \.self) { key in
+                                if let e = childEntities[key] {
+                                    let _ = e.forceUpdate ? 0 : 0
+                                    SpatialModel3DView(parentYOffset: parentYOffset)
+                                        .environment(e)
+                                }
+                            }
+
+                            // SpatialView content
+                            ForEach(Array(childEntities.keys), id: \.self) { key in
+                                if let e = childEntities[key] {
+                                    if e.coordinateSpace == .DOM {
+                                        if let viewComponent = e.getComponent(SpatialViewComponent.self) {
+                                            let _ = e.forceUpdate ? 0 : 0
+                                            let x = CGFloat(e.modelEntity.position.x)
+                                            let y = CGFloat(e.modelEntity.position.y - parentYOffset)
+                                            let z = CGFloat(e.modelEntity.position.z)
+
+                                            let width = CGFloat(viewComponent.resolutionX)
+                                            let height = CGFloat(viewComponent.resolutionY)
+
+                                            SpatialViewUI().environment(e).frame(width: width, height: height).scaleEffect(
+                                                x: CGFloat(e.modelEntity.scale.x),
+                                                y: CGFloat(e.modelEntity.scale.y),
+                                                z: CGFloat(e.modelEntity.scale.z)
+                                            )
+                                            .rotation3DEffect(
+                                                Rotation3D(simd_quatf(
+                                                    ix: e.modelEntity.orientation.vector.x,
+                                                    iy: e.modelEntity.orientation.vector.y,
+                                                    iz: e.modelEntity.orientation.vector.z,
+                                                    r: e.modelEntity.orientation.vector.w
+                                                ))
+                                            ).position(x: x, y: y)
+                                            .offset(z: z)
+                                        }
+                                    }
+                                }
+                            }
+                        }.frame(maxWidth: .infinity, maxHeight: .infinity).frame(maxDepth: 0, alignment: .back).offset(z: 0)
+                    }
+
+                    // Display the main webview
+                    if wv.didFailLoad {
+                        VStack {
+                            Text("Failed to load webpage. Is the server running?")
+                                .foregroundColor(.white)
+                            Button("Reload") {
+                                if let url = wv.getURL() {
+                                    wv.navigateToURL(url: url)
+                                } else {
+                                    logger.warning("Unable to reload URL")
+                                }
+                            }
+                            .foregroundColor(.white)
+                        }
+                        .frame(maxWidth: .infinity, maxHeight: .infinity).glassBackgroundEffect()
+
+                    } else {
+                        wv.getView()
+                            .materialWithBorderCorner(
+                                wv.backgroundMaterial,
+                                wv.cornerRadius
+                            )
+
+                            .frame(maxWidth: .infinity, maxHeight: .infinity)
+                    }
+                }
+                .opacity(wv.opacity)
+                .hidden(!ent.visible)
+            }
+        }
+    }
+}

package/template/visionOSApp/web-spatial/views/VolumetricWindowContainerView.swift

@@ -0,0 +1,34 @@
+//
+//  VolumetricWindowContainerView.swift
+//  web-spatial
+//
+//  Created by ByteDance on 5/9/24.
+//
+
+import typealias RealityKit.Attachment
+import typealias RealityKit.Entity
+import typealias RealityKit.MeshResource
+import typealias RealityKit.Model3D
+import typealias RealityKit.ModelEntity
+import typealias RealityKit.RealityView
+import typealias RealityKit.SimpleMaterial
+import SwiftUI
+
+struct VolumetricWindowContainerView: View {
+    @Environment(SpatialWindowContainer.self) var windowContainerContent: SpatialWindowContainer
+
+    var body: some View {
+        OpenDismissHandlerUI().environment(windowContainerContent).onDisappear {
+            windowContainerContent.destroy()
+        }
+
+        let entities = windowContainerContent.getEntities().filter { _, entity in
+            entity.coordinateSpace == .ROOT && entity.hasComponent(SpatialViewComponent.self)
+        }
+
+        ForEach(Array(entities.keys), id: \.self) { key in
+            let entity = entities[key]!
+            SpatialViewUI(isRoot: true).environment(entity)
+        }
+    }
+}

package/template/visionOSApp/web-spatial/views/ui/NavView.swift

@@ -0,0 +1,88 @@
+//
+//  NavView.swift
+//  web-spatial
+//
+//  Created by ByteDance on 2025/1/8.
+//
+
+import SwiftUI
+import WebKit
+
+struct NavView: View {
+    @State var swc: SpatialWindowComponent?
+    @State var showUrl: Bool = false
+    @State private var showCopyTip = false
+
+    var body: some View {
+        ZStack(alignment: .bottomTrailing) {
+            VStack(alignment: .trailing, spacing: 0) {
+                HStack(spacing: 10) {
+                    Text(pwaManager.name).padding(.trailing, 10)
+                    if pwaManager.display == .minimal {
+                        Button(action: {
+                            swc?.goBack()
+                        }, label: {
+                            Image(systemName: "arrow.left")
+                        })
+                        .disabled(!swc!.canGoBack)
+                        Button(action: {
+                            swc?.goForward()
+                        }, label: {
+                            Image(systemName: "arrow.right")
+                        })
+                        .disabled(!swc!.canGoForward)
+                        Button(action: {
+                            swc?.reload()
+                        }, label: {
+                            Image(systemName: "arrow.clockwise")
+                        })
+                        Button(
+                            action: {
+                                swc?
+                                    .navigateToURL(
+                                        url: URL(string: pwaManager.start_url)!
+                                    )
+                            },
+                            label: {
+                                Image(systemName: "house.fill")
+                            }
+                        )
+                    }
+                    Button(action: {
+                        showUrl.toggle()
+                    }, label: {
+                        Image(systemName: "info.circle")
+                    })
+                }
+            }.padding().glassBackgroundEffect(in: .rect).cornerRadius(15)
+            HStack(spacing: 0) {
+                Text(
+                    swc?.getURL()?.absoluteString ?? ""
+                )
+                .padding()
+                Button(action: {
+                    UIPasteboard.general.string = swc?.getURL()?.absoluteString ?? ""
+                    showCopyTip = true
+                    DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
+                        showCopyTip = false
+                    }
+                }, label: {
+                    Text("copy")
+                }).padding(.trailing, 5)
+
+                Button(action: {
+                    showUrl = false
+                }, label: {
+                    Text("X")
+                })
+            }.glassBackgroundEffect(in: .rect).cornerRadius(15).offset(y: 35).opacity(showUrl ? 1 : 0).animation(.easeOut, value: 0.2)
+        }
+        .zIndex(10) // closer to user
+        .popover(isPresented: $showCopyTip) {
+            Text("copied！")
+                .padding()
+                .background(.ultraThinMaterial)
+                .cornerRadius(10)
+        }
+    }
+}

package/template/visionOSApp/web-spatial/web_spatialApp.swift

@@ -0,0 +1,134 @@
+//
+//  web_spatialApp.swift
+//  web-spatial
+//
+//  Created by ByteDance on 5/8/24.
+//
+
+import typealias RealityKit.Attachment
+import typealias RealityKit.Entity
+import typealias RealityKit.MeshResource
+import typealias RealityKit.Model3D
+import typealias RealityKit.ModelEntity
+import typealias RealityKit.RealityView
+import typealias RealityKit.SimpleMaterial
+import SwiftUI
+
+let logger = Logger()
+
+// To load a local path, remove http:// eg.  "static-web/"
+let nativeAPIVersion = "0.0.1"
+
+// detect when app properties like defaultSize change so we can avoid race condition of setting default values and then opening window container
+var sceneStateChangedCB: ((Any) -> Void) = { _ in
+}
+
+@main
+struct web_spatialApp: App {
+    @UIApplicationDelegateAdaptor(AppDelegate.self) var appDelegate
+    @State var root: SpatialWindowComponent? = nil
+    @State var rootWGD: SpatialWindowContainer
+    @State var initialLaunch = true
+
+    @ObservedObject var wgm = WindowContainerMgr.Instance
+
+    @Environment(\.scenePhase) private var scenePhase
+
+    init() {
+        logger.debug("WebSpatial App Started --------")
+
+        // init global logger
+        Logger.initLogger()
+
+        // init pwa manager
+        pwaManager._init()
+
+        // create root SpatialWindowContainer and Immersive SpatialWindowContainer
+        rootWGD = SpatialWindowContainer.createRootWindowContainer()
+        let _ = SpatialWindowContainer.createImmersiveWindowContainer()
+    }
+
+    func getFileUrl() -> URL {
+        return URL(string: pwaManager.start_url)!
+    }
+
+    // There seems to be a bug in WKWebView where it needs to be initialized after the app has loaded so we do this here instead of init()
+    // https://forums.developer.apple.com/forums/thread/61432
+    func initAppOnViewMount() {
+        if root == nil {
+            let fileUrl = getFileUrl()
+
+            // Create a default entity with webview resource
+            let rootEntity = SpatialEntity()
+            rootEntity.coordinateSpace = CoordinateSpaceMode.ROOT
+            let windowComponent = SpatialWindowComponent(parentWindowContainerID: rootWGD.id, url: fileUrl)
+            rootEntity.addComponent(windowComponent)
+            rootEntity.setParentWindowContainer(wg: rootWGD)
+
+            root = windowComponent
+        }
+    }
+
+    func getDefaultSize() -> CGSize {
+        sceneStateChangedCB("")
+        return wgm.getValue().defaultSize!
+    }
+
+    var body: some Scene {
+        WindowGroup(id: "Plain", for: WindowContainerData.self) { $windowData in
+            if windowData.windowContainerID == SpatialWindowContainer.getRootID() {
+                VStack {}.onAppear { initAppOnViewMount() }
+
+                PlainWindowContainerView().environment(rootWGD).background(Color.clear.opacity(0)).onOpenURL { myURL in
+                    initAppOnViewMount()
+                    let urlToLoad = pwaManager.checkInDeeplink(url: myURL.absoluteString)
+
+                    if let url = URL(string: urlToLoad) {
+                        root!.navigateToURL(url: url)
+                    }
+                }
+            } else {
+                let wg = SpatialWindowContainer.getOrCreateSpatialWindowContainer(
+                    windowData.windowContainerID, windowData
+                )
+                PlainWindowContainerView().environment(wg)
+                    // https://stackoverflow.com/questions/78567737/how-to-get-initial-windowgroup-to-reopen-on-launch-visionos
+                    .handlesExternalEvents(preferring: [], allowing: [])
+            }
+        } defaultValue: {
+            WindowContainerData(windowStyle: "Plain", windowContainerID: SpatialWindowContainer.getRootID())
+
+        }.windowStyle(.plain).onChange(of: scenePhase) { oldPhase, newPhase in
+            if oldPhase == .background && newPhase == .inactive {
+                if initialLaunch {
+                    // App initial open
+                    initialLaunch = false
+                } else {
+                    // App reopened
+                    let fileUrl = getFileUrl()
+                    root?.navigateToURL(url: fileUrl)
+                    rootWGD.setSize.send(DefaultPlainWindowContainerSize)
+                }
+            }
+        }.defaultSize(
+            getDefaultSize()
+        ).windowResizability(
+            wgm.getValue().windowResizability!
+        )
+
+        WindowGroup(id: "Volumetric", for: WindowContainerData.self) { $windowData in
+            let wg = SpatialWindowContainer.getOrCreateSpatialWindowContainer(windowData!.windowContainerID, windowData!)
+            VolumetricWindowContainerView().environment(wg).handlesExternalEvents(preferring: [], allowing: [])
+
+        }.windowStyle(.volumetric).defaultSize(width: 1, height: 1, depth: 1, in: .meters)
+
+        ImmersiveSpace(id: "ImmersiveSpace") {
+            let wg = SpatialWindowContainer.getImmersiveWindowContainer()
+            VolumetricWindowContainerView().environment(wg).handlesExternalEvents(preferring: [], allowing: [])
+        }
+
+        WindowGroup(id: "loading") {
+            LoadingView()
+        }
+    }
+}