@webspatial/platform-visionos 1.3.0 → 1.4.0

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "@webspatial/platform-visionos",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Used to publish WebSpatial projects to Apple Vision Pro",
   "type": "commonjs",
+  "main": "package.json",
   "engines": {
     "node": ">=14.15.0"
   },

package/web-spatial/JSBCommand.swift

@@ -129,6 +129,13 @@ struct ConvertFromSceneToEntity: CommandDataProtocol {
     let position: Vec3
 }
 
+struct ConvertCoordinate: CommandDataProtocol {
+    static let commandType: String = "ConvertCoordinate"
+    let position: Vec3
+    let fromId: String
+    let toId: String
+}
+
 struct InspectCommand: CommandDataProtocol {
     static let commandType: String = "Inspect"
     var id: String?
@@ -166,6 +173,7 @@ protocol SpatializedElementProperties: SpatialObjectCommand {
     var enableMagnifyGesture: Bool? { get }
     var enableMagnifyEndGesture: Bool? { get }
     var enableTapGesture: Bool? { get }
+    var rotateConstrainedToAxis: Vec3? { get }
 }
 
 struct UpdateSpatialized2DElementProperties: SpatializedElementProperties {
@@ -193,6 +201,8 @@ struct UpdateSpatialized2DElementProperties: SpatializedElementProperties {
     var enableMagnifyEndGesture: Bool?
     var enableTapGesture: Bool?
 
+    let rotateConstrainedToAxis: Vec3?
+
     let scrollPageEnabled: Bool?
     let material: BackgroundMaterial?
     let cornerRadius: CornerRadius?
@@ -227,6 +237,8 @@ struct UpdateSpatializedStatic3DElementProperties: SpatializedElementProperties
     let enableMagnifyEndGesture: Bool?
     let enableTapGesture: Bool?
 
+    let rotateConstrainedToAxis: Vec3?
+
     let modelURL: String?
     let modelTransform: [Double]?
 }
@@ -255,6 +267,8 @@ struct UpdateSpatializedDynamic3DElementProperties: SpatializedElementProperties
     let enableMagnifyGesture: Bool?
     let enableMagnifyEndGesture: Bool?
     let enableTapGesture: Bool?
+
+    let rotateConstrainedToAxis: Vec3?
 }
 
 struct UpdateSpatializedElementTransform: SpatialObjectCommand {
@@ -339,6 +353,15 @@ struct GetSpatialSceneStateCommand: CommandDataProtocol {
     static let commandType = "GetSpatialSceneState"
 }
 
+struct InitializeAttachmentCommand: CommandDataProtocol {
+    static let commandType = "InitializeAttachment"
+    let id: String
+    let parentEntityId: String
+    let position: [Float]?
+    let size: AttachmentSize?
+    let ownerViewId: String
+}
+
 struct UpdateAttachmentEntityCommand: CommandDataProtocol {
     static let commandType = "UpdateAttachmentEntity"
     let id: String

package/web-spatial/WebMsgCommand.swift

@@ -14,8 +14,6 @@ enum WebSpatialGestureType: String, Encodable {
 }
 
 enum SpatialWebMsgType: String, Encodable {
-    case cubeInfo
-    case transform
     case modelloaded
     case modelloadfailed
     case spatialtap
@@ -30,19 +28,6 @@ enum SpatialWebMsgType: String, Encodable {
     case objectdestroy
 }
 
-/// notify Spatialized3DElement Container Cube, used for ref.current.getBoundingClientCube()
-struct SpatiaizedContainerClientCube: Encodable {
-    let type: SpatialWebMsgType = .cubeInfo
-    let origin: Point3D
-    let size: Size3D
-}
-
-/// notify Spatialized3DElement Container Transform to SpatialScene, used for ref.current.convertToSpatialScene()
-struct SpatiaizedContainerTransform: Encodable {
-    let type: SpatialWebMsgType = .transform
-    let detail: AffineTransform3D
-}
-
 struct WebSpatialTapGuestureEventDetail: Encodable {
     let location3D: Point3D
     /// Global scene location (maps to clientX/clientY/clientZ on the web side).

package/web-spatial/manager/AttachmentManager.swift

@@ -9,7 +9,7 @@ struct AttachmentInfo: Identifiable, Equatable {
     var webViewModel: SpatialWebViewModel
 
     static func == (lhs: AttachmentInfo, rhs: AttachmentInfo) -> Bool {
-        return lhs.id == rhs.id
+        lhs.id == rhs.id
     }
 }
 
@@ -29,9 +29,9 @@ class AttachmentManager {
         id: String,
         parentEntityId: String,
         position: SIMD3<Float>,
-        size: CGSize
+        size: CGSize,
+        webViewModel: SpatialWebViewModel
     ) -> AttachmentInfo {
-        let webViewModel = SpatialWebViewModel(url: nil)
         webViewModel.setBackgroundTransparent(true)
         // webViewModel.scrollEnabled = false
 
@@ -48,12 +48,14 @@ class AttachmentManager {
 
     func update(id: String, position: SIMD3<Float>?, size: CGSize?) {
         guard var info = attachments[id] else { return }
+
         if let position = position {
             info.position = position
         }
         if let size = size {
             info.size = size
         }
+
         attachments[id] = info
     }
 
@@ -66,12 +68,13 @@ class AttachmentManager {
     }
 
     func get(id: String) -> AttachmentInfo? {
-        return attachments[id]
+        attachments[id]
     }
 
     func destroyAll() {
         let toDestroy = Array(attachments.values)
         attachments.removeAll()
+
         DispatchQueue.main.async {
             for info in toDestroy {
                 info.webViewModel.destroy()

package/web-spatial/manager/Dynamic3DManager.swift

@@ -71,6 +71,16 @@ class Dynamic3DManager {
     }
 
     static func loadResourceToLocal(_ urlString: String, loadComplete: @escaping (Result<URL, Error>) -> Void) {
+        // load local file
+        if urlString.starts(with: "file://") {
+            guard let localUrl = URL(string: pwaManager.getLocalResourceURL(url: urlString)) else {
+                loadComplete(.failure(NSError(domain: "Download Error", code: 0, userInfo: [NSLocalizedDescriptionKey: "Local file is not found"])))
+                return
+            }
+            loadComplete(.success(localUrl))
+            return
+        }
+        // load net file
         guard let url = URL(string: urlString) else {
             loadComplete(.failure(NSError(domain: "Invalid URL", code: 0, userInfo: [NSLocalizedDescriptionKey: "Failed to create URL from string: \(urlString)"])))
             return

package/web-spatial/manager/WKWebViewManager.swift

@@ -8,8 +8,8 @@ class WKWebViewManager {
 
     func create(controller: SpatialWebController, configuration: WKWebViewConfiguration? = nil, spatialId: String? = "") -> WKWebView {
         let userContentController = WKUserContentController()
-        // TODO: get native api instead of PACKAGE_VERSION
-        let userScript = WKUserScript(source: "window.WebSpatailEnabled = true; window.WebSpatailNativeVersion = 'PACKAGE_VERSION';", injectionTime: .atDocumentStart, forMainFrameOnly: false)
+        // TODO: get native api instead of using the injected WS_SDK_VERSION placeholder
+        let userScript = WKUserScript(source: "window.WebSpatailEnabled = true; window.WebSpatailNativeVersion = 'WS_SDK_VERSION';", injectionTime: .atDocumentStart, forMainFrameOnly: false)
         userContentController.addUserScript(userScript)
 //        userContentController.add(controller, name: "bridge")
         userContentController.addScriptMessageHandler(controller, contentWorld: .page, name: "bridge")
@@ -25,8 +25,8 @@ class WKWebViewManager {
         // change webview ua
         let ua = controller.webview!.value(forKey: "userAgent") as? String ?? ""
         let webviewVersion = ua.split(separator: configUA)[0].split(separator: "AppleWebKit")[1]
-        // TODO: get native api instead of PACKAGE_VERSION
-        controller.webview!.customUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; wv) AppleWebKit\(webviewVersion)WebSpatial/\("PACKAGE_VERSION") SpatialID/\(spatialId!)"
+        // TODO: get native api instead of relying on injected shell/sdk versions
+        controller.webview!.customUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; wv) AppleWebKit\(webviewVersion)WSAppShell/\(pwaManager.getShellVersion()) WebSpatial/\(pwaManager.getSdkVersion()) SpatialID/\(spatialId!)"
         controller.webview!.uiDelegate = controller
         controller.webview!.allowsBackForwardNavigationGestures = false
         controller.webview!.isInspectable = true

package/web-spatial/manifest.swift

@@ -6,9 +6,9 @@ var pwaManager = PWAManager()
 struct PWAManager: Codable {
     var isLocal: Bool = false
 
-    var start_url: String = "http://localhost:5173"
+    var start_url: String = "http://localhost:5173/#/geometry-verify"
 
-//    var start_url: String = "http://localhost:5173/webspatial/avp/materialApiTest"
+    // var start_url: String = "http://localhost:5173/#/spatial-drag-gesture"
 
     var scope: String = ""
     var id: String = "com.webspatial.pico"
@@ -38,7 +38,8 @@ struct PWAManager: Codable {
         baseplateVisibility: nil
     )
     var useMainScene: Bool = true
-    private var version: String = "PACKAGE_VERSION"
+    private var shellVersion: String = "WS_SHELL_VERSION"
+    private var sdkVersion: String = "WS_SDK_VERSION"
 
     mutating func _init() {
         let urlType = start_url.split(separator: "://").first
@@ -102,8 +103,12 @@ struct PWAManager: Codable {
         return resource
     }
 
-    func getVersion() -> String {
-        return version
+    func getShellVersion() -> String {
+        return shellVersion
+    }
+
+    func getSdkVersion() -> String {
+        return sdkVersion
     }
 }
 

package/web-spatial/model/SpatialApp.swift

@@ -4,7 +4,7 @@ import SwiftUI
 let logger = Logger()
 
 /// To load a local path, remove http:// eg.  "static-web/"
-let nativeAPIVersion = pwaManager.getVersion()
+let nativeAPIVersion = pwaManager.getShellVersion()
 
 /// start URL
 let startURL = pwaManager.start_url
@@ -84,7 +84,7 @@ class SpatialApp {
     }
 
     var version: String {
-        pwaManager.getVersion()
+        pwaManager.getShellVersion()
     }
 
     var startURL: String {

package/web-spatial/model/SpatialScene.swift

@@ -1,5 +1,6 @@
 import Combine
 import Foundation
+import RealityKit
 import simd
 import SwiftUI
 
@@ -86,6 +87,9 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
 
     var spatialWebViewModel: SpatialWebViewModel
 
+    private var meterToPtUnscaled: Double?
+    private var meterToPtScaled: Double?
+
     init(
         _ url: String,
         _ windowStyle: WindowStyle,
@@ -108,6 +112,20 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         spatialWebViewModel.sendWebEvent(id, msg)
     }
 
+    func onUpdatePhysicalMetrics(meterToPtUnscaled: Double, meterToPtScaled: Double) {
+        self.meterToPtUnscaled = meterToPtUnscaled
+        self.meterToPtScaled = meterToPtScaled
+        let js = """
+        window.__webspatialsdk__ = window.__webspatialsdk__ || {};
+        window.__webspatialsdk__.physicalMetrics = {
+            meterToPtUnscaled: \(meterToPtUnscaled),
+            meterToPtScaled: \(meterToPtScaled)
+        };
+        """
+        spatialWebViewModel.getController().callJS(js)
+        sendWebMsg("window", "")
+    }
+
     private func setupSpatialWebView() {
         setupJSBListeners()
         setupWebViewStateListener()
@@ -300,6 +318,8 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         spatialWebViewModel.addJSBListener(ConvertFromEntityToEntity.self, onConvertFromEntityToEntity)
         spatialWebViewModel.addJSBListener(ConvertFromEntityToScene.self, onConvertFromEntityToScene)
         spatialWebViewModel.addJSBListener(ConvertFromSceneToEntity.self, onConvertFromSceneToEntity)
+        spatialWebViewModel.addJSBListener(InitializeAttachmentCommand.self, onInitializeAttachment)
+        spatialWebViewModel.addJSBListener(ConvertCoordinate.self, onConvertCoordinate)
 
         spatialWebViewModel.addJSBListener(UpdateAttachmentEntityCommand.self, onUpdateAttachmentEntity)
 
@@ -326,8 +346,8 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
 
         // write through
         spatialWebViewModel.updateWindowKV([
-            "innerDepth": depth,
-            "outerDepth": depth,
+            "xrInnerDepth": depth,
+            "xrOuterDepth": depth,
             "outerHeight": height + SpatialScene.navHeight,
         ])
     }
@@ -349,6 +369,17 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
             self.handleWindowClose()
         }
 
+        spatialWebViewModel.addStateListener(.didReceive) {
+            if let meterToPtUnscaled = self.meterToPtUnscaled,
+               let meterToPtScaled = self.meterToPtScaled
+            {
+                self.onUpdatePhysicalMetrics(
+                    meterToPtUnscaled: meterToPtUnscaled,
+                    meterToPtScaled: meterToPtScaled
+                )
+            }
+        }
+
         spatialWebViewModel.addStateListener(.didFailLoad) {
             self.didFailLoad = true
         }
@@ -394,46 +425,51 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         }
     }
 
+    // Temporary storage for webview models awaiting JSB initialization
+    private var pendingAttachmentWebViewModels = [String: SpatialWebViewModel]()
+
     private func handleCreateAttachment(_ url: URL) -> WebViewElementInfo? {
-        guard let components = URLComponents(string: url.absoluteString),
-              let queryItems = components.queryItems
-        else {
-            print("❌ fail to parse attachment URL")
-            return nil
-        }
+        // Just create a bare webview — metadata arrives via InitializeAttachment JSB
+        let id = UUID().uuidString
+        let webViewModel = SpatialWebViewModel(url: nil)
+        webViewModel.setBackgroundTransparent(true)
+        pendingAttachmentWebViewModels[id] = webViewModel
+        return WebViewElementInfo(id: id, element: webViewModel)
+    }
 
-        guard let parentEntityId = queryItems.first(where: { $0.name == "parentEntityId" })?.value else {
-            print("❌ missing parentEntityId for attachment")
-            return nil
+    private func onInitializeAttachment(
+        command: InitializeAttachmentCommand,
+        resolve: @escaping JSBManager.ResolveHandler<Encodable>
+    ) {
+        guard let webViewModel = pendingAttachmentWebViewModels.removeValue(forKey: command.id) else {
+            resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "No pending attachment for \(command.id)")))
+            return
         }
 
-        // Parse position (JSON array like [0,0.1,0])
         var position = SIMD3<Float>(0, 0, 0)
-        if let positionStr = queryItems.first(where: { $0.name == "position" })?.value?.removingPercentEncoding,
-           let positionData = positionStr.data(using: .utf8),
-           let positionArray = try? JSONDecoder().decode([Float].self, from: positionData),
-           positionArray.count >= 3
-        {
-            position = SIMD3<Float>(positionArray[0], positionArray[1], positionArray[2])
+        if let posArray = command.position, posArray.count >= 3 {
+            position = SIMD3<Float>(posArray[0], posArray[1], posArray[2])
         }
 
-        // Parse size (JSON object like {"width":100,"height":100})
-        var size = CGSize(width: 100, height: 100)
-        if let sizeStr = queryItems.first(where: { $0.name == "size" })?.value?.removingPercentEncoding,
-           let sizeData = sizeStr.data(using: .utf8),
-           let sizeObj = try? JSONDecoder().decode(AttachmentSize.self, from: sizeData)
-        {
-            size = CGSize(width: sizeObj.width, height: sizeObj.height)
-        }
+        let size = CGSize(
+            width: command.size?.width ?? 100,
+            height: command.size?.height ?? 100
+        )
 
-        let info = attachmentManager.create(
-            id: UUID().uuidString,
-            parentEntityId: parentEntityId,
+        let ownerId = command.ownerViewId
+        if spatialObjects[ownerId] == nil {
+            resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "ownerViewId must belong to the current scene for attachment \(command.id)")))
+            return
+        }
+        attachmentManager.create(
+            id: command.id,
+            parentEntityId: command.parentEntityId,
             position: position,
-            size: size
+            size: size,
+            webViewModel: webViewModel
         )
 
-        return WebViewElementInfo(id: info.id, element: info.webViewModel)
+        resolve(.success(baseReplyData))
     }
 
     private func onPageStartLoad() {
@@ -442,7 +478,6 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         for spatialObject in spatialObjectArray {
             spatialObject.destroy()
         }
-        // destroy all attachments
         attachmentManager.destroyAll()
         backgroundMaterial = .None
     }
@@ -715,6 +750,10 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
             spatializedElement.enableRotateEndGesture = enableRotateEndGesture
         }
 
+        if let rotateConstrainedToAxis = command.rotateConstrainedToAxis {
+            spatializedElement.rotateConstrainedToAxis = rotateConstrainedToAxis
+        }
+
         if let enableMagnifyGesture = command.enableMagnifyGesture {
             spatializedElement.enableMagnifyGesture = enableMagnifyGesture
         }
@@ -1068,22 +1107,91 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         resolve(.success(ConvertReply(id: command.entityId, position: point)))
     }
 
+    /// Input: command.position, command.fromId, command.toId
+    /// fromId/toId can reference either the scene (window) or an entity.
+    /// Step 1: Convert position to window coordinates (view global, px)
+    ///   - If from is window (scene), position is already in view global (px). Go to Step 2.
+    ///   - If from is 2d frame(SpatializedElement), position is in view local (px).
+    ///     - view local → window (view global, px) using SpatializedElement.convertToScene
+    ///   - If from is an entity, position is in reality entity local (meters):
+    ///     - entity local → reality world (scene)
+    ///     - reality world → window (view global, px)
+    /// Step 2: Convert window coordinates (view global, px) to target output
+    ///   - If to is window, output directly.
+    ///   - If to is 2d frame(SpatializedElement), output in view local (px).
+    ///     - window (view global, px) → view local using SpatializedElement.convertFromScene
+    ///   - If to is an entity, output in reality entity local (meters):
+    ///     - window (view global, px) → reality world (scene)
+    ///     - reality world → reality entity local (meters)
+
+    private func onConvertCoordinate(command: ConvertCoordinate, resolve: @escaping JSBManager.ResolveHandler<Encodable>) {
+        func isSceneId(_ id: String) -> Bool {
+            return id.isEmpty
+        }
+        let input = SIMD3<Float>(Float(command.position.x), Float(command.position.y), Float(command.position.z))
+        let fromEntity = spatialObjects[command.fromId] as? SpatialEntity
+        let from2dFrame = spatialObjects[command.fromId] as? SpatializedElement
+        let toEntity = spatialObjects[command.toId] as? SpatialEntity
+        let to2dFrame = spatialObjects[command.toId] as? SpatializedElement
+
+        var globalPx: Point3D
+        if isSceneId(command.fromId) {
+            globalPx = Point3D(x: Double(input.x), y: Double(input.y), z: Double(input.z))
+        } else if let fromEntity {
+            let world = fromEntity.convert(position: input, to: nil)
+            guard let content = findSpatializedDynamic3DElement(containingEntityId: fromEntity.spatialId)?.getViewContent() else {
+                resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "RealityView content unavailable for conversion")))
+                return
+            }
+            globalPx = content.convert(point: world, from: .scene, to: .global)
+        } else if let from2dFrame {
+            let localPoint = SIMD3<Double>(Double(input.x), Double(input.y), Double(input.z))
+            let scenePoint = from2dFrame.convertToScene(localPoint)
+            globalPx = Point3D(x: scenePoint.x, y: scenePoint.y, z: scenePoint.z)
+        } else {
+            resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "Invalid fromId")))
+            return
+        }
+
+        if isSceneId(command.toId) {
+            let result = Vec3(x: CGFloat(globalPx.x), y: CGFloat(globalPx.y), z: CGFloat(globalPx.z))
+            resolve(.success(result))
+            return
+        } else if let toEntity {
+            guard let content = findSpatializedDynamic3DElement(containingEntityId: toEntity.spatialId)?.getViewContent() else {
+                resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "RealityView content unavailable for conversion")))
+                return
+            }
+            let world = content.convert(globalPx, from: .global, to: .scene)
+            let local = toEntity.convert(position: world, from: nil)
+            let ret = Vec3(x: CGFloat(local.x), y: CGFloat(local.y), z: CGFloat(local.z))
+            resolve(.success(ret))
+            return
+        } else if let to2dFrame {
+            let scenePoint = SIMD3<Double>(globalPx.x, globalPx.y, globalPx.z)
+            let localPoint = to2dFrame.convertFromScene(scenePoint)
+            let ret = Vec3(x: CGFloat(localPoint.x), y: CGFloat(localPoint.y), z: CGFloat(localPoint.z))
+            resolve(.success(ret))
+            return
+        } else {
+            resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "Invalid toId")))
+            return
+        }
+    }
+
     private func onUpdateAttachmentEntity(command: UpdateAttachmentEntityCommand, resolve: @escaping JSBManager.ResolveHandler<Encodable>) {
         guard attachmentManager.get(id: command.id) != nil else {
             resolve(.failure(JsbError(code: .InvalidSpatialObject, message: "Attachment \(command.id) not found")))
             return
         }
-
         var newPosition: SIMD3<Float>? = nil
         if let posArray = command.position, posArray.count >= 3 {
             newPosition = SIMD3<Float>(posArray[0], posArray[1], posArray[2])
         }
-
         var newSize: CGSize? = nil
         if let sizeObj = command.size {
             newSize = CGSize(width: sizeObj.width, height: sizeObj.height)
         }
-
         attachmentManager.update(id: command.id, position: newPosition, size: newSize)
         resolve(.success(baseReplyData))
     }
@@ -1111,6 +1219,32 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         sendWebMsg(spatialObject.spatialId, SpatialObjectDestroiedEvent())
     }
 
+    /// Find the dynamic 3D container (SpatializedDynamic3DElement) that contains the entity by ID.
+    /// - Parameter entityId: The entity's spatialId.
+    /// - Returns: The container if the entity is a descendant of the container's root; otherwise nil.
+    private func findSpatializedDynamic3DElement(containingEntityId entityId: String) -> SpatializedDynamic3DElement? {
+        guard let entity = spatialObjects[entityId] as? SpatialEntity else {
+            return nil
+        }
+
+        for (_, object) in spatialObjects {
+            guard let dynamic3dElement = object as? SpatializedDynamic3DElement else {
+                continue
+            }
+
+            let root = dynamic3dElement.getRoot()
+            var current: Entity? = entity
+            while let node = current {
+                if node === root {
+                    return dynamic3dElement
+                }
+                current = node.parent
+            }
+        }
+
+        return nil
+    }
+
     func findSpatialObject<T: SpatialObjectProtocol>(_ id: String) -> T? {
         return spatialObjects[id] as? T
     }

package/web-spatial/model/SpatializedDynamic3DElement.swift

@@ -1,8 +1,11 @@
+import _RealityKit_SwiftUI
 import Foundation
+import RealityKit
 
 @Observable
 class SpatializedDynamic3DElement: SpatializedElement {
     private var rootEntity = SpatialEntity()
+    private var viewContent: RealityViewContent? = nil
 
     func getRoot() -> SpatialEntity {
         return rootEntity
@@ -16,6 +19,14 @@ class SpatializedDynamic3DElement: SpatializedElement {
         rootEntity.removeChild(entity)
     }
 
+    func getViewContent() -> RealityViewContent? {
+        return viewContent
+    }
+
+    func setViewContent(_ content: RealityViewContent?) {
+        viewContent = content
+    }
+
     enum CodingKeys: String, CodingKey {
         case type, root
     }
@@ -28,6 +39,7 @@ class SpatializedDynamic3DElement: SpatializedElement {
     }
 
     override func onDestroy() {
+        viewContent = nil
         rootEntity.destroy()
         super.onDestroy()
     }

package/web-spatial/model/SpatializedElement.swift

@@ -2,6 +2,9 @@ import Foundation
 import RealityKit
 import SwiftUI
 
+/// zIndex() have some bug, so use zOrderBias to simulate zIndex effect
+let zOrderBias = 0.001
+
 enum SpatializedElementType: String, Codable {
     case Spatialized2DElement
     case SpatializedStatic3DElement
@@ -32,8 +35,45 @@ class SpatializedElement: SpatialObject {
     var enableMagnifyEndGesture: Bool = false
     var enableTapGesture: Bool = false
 
+    /// When non-nil and non-zero length, rotate gesture is constrained to this axis (world space).
+    var rotateConstrainedToAxis: Vec3?
+
     var defaultAlignment: DepthAlignment = .back
 
+    /// Raw layout→scene transform from onGeometryChange3D proxy.
+    /// Does NOT include backOffset or zIndex offset.
+    /// Updated by SpatializedElementView whenever layout changes.
+    var proxySceneTransform: AffineTransform3D = .identity
+
+    /// Full local→scene transform accounting for --xr-back and zIndex.
+    /// Computed on-the-fly so backOffset/zIndex changes are always reflected.
+    var sceneTransform: AffineTransform3D {
+        let frameZ = (zIndex * zOrderBias) + backOffset
+        let localZ = AffineTransform3D(translation: Vector3D(x: 0, y: 0, z: frameZ))
+        return proxySceneTransform.concatenating(localZ)
+    }
+
+    /// Converts a point from this element's local coordinate system to scene space.
+    func convertToScene(_ localPoint: SIMD3<Double>) -> SIMD3<Double> {
+        let p = SIMD4<Double>(localPoint.x, localPoint.y, localPoint.z, 1.0)
+        let scene = sceneTransform.matrix * p
+        return SIMD3<Double>(scene.x, scene.y, scene.z)
+    }
+
+    /// Converts a point from scene space to this element's local coordinate system.
+    func convertFromScene(_ scenePoint: SIMD3<Double>) -> SIMD3<Double> {
+        let inv = sceneTransform.inverse!
+        let p = SIMD4<Double>(scenePoint.x, scenePoint.y, scenePoint.z, 1.0)
+        let local = inv.matrix * p
+        return SIMD3<Double>(local.x, local.y, local.z)
+    }
+
+    /// Converts a point from this element's local space to another element's local space.
+    func convert(_ localPoint: SIMD3<Double>, to target: SpatializedElement) -> SIMD3<Double> {
+        let scenePoint = convertToScene(localPoint)
+        return target.convertFromScene(scenePoint)
+    }
+
     var enableGesture: Bool {
         return enableDragStartGesture || enableDragGesture || enableDragEndGesture || enableRotateGesture || enableRotateEndGesture || enableMagnifyGesture || enableMagnifyEndGesture || enableTapGesture
     }

package/web-spatial/model/dynamic3d/SpatialEntity.swift

@@ -138,10 +138,16 @@ class SpatialEntity: Entity, SpatialObjectProtocol {
             if !components.has(InputTargetComponent.self) {
                 components.set(InputTargetComponent())
             }
+            if !components.has(HoverEffectComponent.self) {
+                components.set(HoverEffectComponent())
+            }
         } else {
             if components.has(InputTargetComponent.self) {
                 components.remove(InputTargetComponent.self)
             }
+            if components.has(HoverEffectComponent.self) {
+                components.remove(HoverEffectComponent.self)
+            }
         }
     }
 

package/web-spatial/view/SceneHandlerUIView.swift

@@ -10,6 +10,9 @@ struct SceneHandlerUIView: View {
     @State var spatialScene: SpatialScene
 
     @Environment(\.scenePhase) private var scenePhase
+    @Environment(\.physicalMetrics) private var converter
+    @State private var latestScaled: Double?
+    @State private var latestUnscaled: Double?
 
     private func setResizibility(resizingRestrictions: UIWindowScene.ResizingRestrictions) {
         sceneDelegate.window?.windowScene?
@@ -41,10 +44,35 @@ struct SceneHandlerUIView: View {
         }
     }
 
+    private func updatePhysicalMetricsIfReady() {
+        if let scaled = latestScaled, let unscaled = latestUnscaled {
+            spatialScene.onUpdatePhysicalMetrics(meterToPtUnscaled: unscaled, meterToPtScaled: scaled)
+        }
+    }
+
     var body: some View {
+        let meterToPtScaled = converter.worldScalingCompensation(.scaled).convert(
+            1,
+            from: .meters
+        )
+        let meterToPtUnscaled = converter.worldScalingCompensation(.unscaled).convert(
+            1,
+            from: .meters
+        )
         VStack {}
             .onAppear {
-                // window scene only resize logic
+                latestScaled = meterToPtScaled
+                latestUnscaled = meterToPtUnscaled
+                updatePhysicalMetricsIfReady()
+            }
+            .onChange(of: meterToPtScaled) { _, newValue in
+                latestScaled = newValue
+                updatePhysicalMetricsIfReady()
+            }
+            .onChange(of: meterToPtUnscaled) { _, newValue in
+                latestUnscaled = newValue
+                updatePhysicalMetricsIfReady()
+            }.onAppear {
                 guard spatialScene.windowStyle == .window else {
                     return
                 }

package/web-spatial/view/SpatializedDynamic3DView.swift

@@ -37,7 +37,7 @@ struct SpatializedDynamic3DView: View {
     }
 
     var rotate3dEvent: some Gesture {
-        RotateGesture3D().targetedToAnyEntity().onChanged { value in
+        makeRotateGesture3D().targetedToAnyEntity().onChanged { value in
             // Always forward rotate gesture events to JS
             if let entity = value.entity as? SpatialEntity {
                 let gestureEvent = WebSpatialRotateGuestureEvent(
@@ -62,6 +62,21 @@ struct SpatializedDynamic3DView: View {
         }
     }
 
+    private func makeRotateGesture3D() -> RotateGesture3D {
+        guard let raw = spatializedElement.rotateConstrainedToAxis else {
+            return RotateGesture3D()
+        }
+        let dx = Double(raw.x)
+        let dy = Double(raw.y)
+        let dz = Double(raw.z)
+        let len = (dx * dx + dy * dy + dz * dz).squareRoot()
+        if len < 1e-9 {
+            return RotateGesture3D()
+        }
+        let axis = RotationAxis3D(x: dx / len, y: dy / len, z: dz / len)
+        return RotateGesture3D(constrainedToAxis: axis)
+    }
+
     var magnifyEvent: some Gesture {
         MagnifyGesture().targetedToAnyEntity().onChanged { value in
             // Always forward magnify gesture events to JS
@@ -122,6 +137,7 @@ struct SpatializedDynamic3DView: View {
         RealityView(make: { content, attachments in
             let rootEntity = spatializedDynamic3DElement.getRoot()
             content.add(rootEntity)
+            spatializedDynamic3DElement.setViewContent(content)
 
             // Add existing attachments on initial creation
             for (_, info) in spatialScene.attachmentManager.attachments {
@@ -168,6 +184,9 @@ struct SpatializedDynamic3DView: View {
         .simultaneousGesture(rotate3dEvent)
         .simultaneousGesture(dragEvent)
         .simultaneousGesture(magnifyEvent)
+        .onDisappear {
+            spatializedDynamic3DElement.setViewContent(nil)
+        }
     }
 
     private func findSpatialEntity(_ spatialId: String) -> SpatialEntity? {

package/web-spatial/view/SpatializedElementView.swift

@@ -1,15 +1,9 @@
 import CoreGraphics
 import SwiftUI
 
-/// zIndex() have some bug, so use zOrderBias to simulate zIndex effect
-let zOrderBias = 0.001
-
-final class GestureFlags {
+final class GestureState {
     var isDrag = false
-}
-
-final class TransformHolder {
-    var transform: AffineTransform3D = .identity
+    var proxyTransform: AffineTransform3D = .identity
 }
 
 struct SpatializedElementView<Content: View>: View {
@@ -19,9 +13,7 @@ struct SpatializedElementView<Content: View>: View {
     var parentScrollOffset: Vec2
     var content: Content
 
-    @State private var gestureFlags = GestureFlags()
-    @State private var transformHolder = TransformHolder()
-    @State private var currentTransform: AffineTransform3D = .identity
+    @State private var gestureState = GestureState()
 
     init(parentScrollOffset: Vec2, @ViewBuilder content: () -> Content) {
         self.parentScrollOffset = parentScrollOffset
@@ -34,7 +26,7 @@ struct SpatializedElementView<Content: View>: View {
             .onChanged(onDragging)
             .onEnded(onDraggingEnded)
             .simultaneously(with:
-                RotateGesture3D()
+                makeRotateGesture3D()
                     .onChanged(onRotateGesture3D)
                     .onEnded(onRotateGesture3DEnd))
             .simultaneously(with:
@@ -46,6 +38,21 @@ struct SpatializedElementView<Content: View>: View {
                     .onEnded(onTapEnded))
     }
 
+    private func makeRotateGesture3D() -> RotateGesture3D {
+        guard let raw = spatializedElement.rotateConstrainedToAxis else {
+            return RotateGesture3D()
+        }
+        let dx = Double(raw.x)
+        let dy = Double(raw.y)
+        let dz = Double(raw.z)
+        let len = (dx * dx + dy * dy + dz * dz).squareRoot()
+        if len < 1e-9 {
+            return RotateGesture3D()
+        }
+        let axis = RotationAxis3D(x: dx / len, y: dy / len, z: dz / len)
+        return RotateGesture3D(constrainedToAxis: axis)
+    }
+
     private func onRotateGesture3D(_ event: RotateGesture3D.Value) {
         if spatializedElement.enableRotateGesture {
             let quaternion = event.rotation.quaternion
@@ -67,16 +74,18 @@ struct SpatializedElementView<Content: View>: View {
     }
 
     private func onDragging(_ event: DragGesture.Value) {
-        if spatializedElement.enableDragStartGesture, !gestureFlags.isDrag {
-            let localPoint = SIMD4<Double>(event.startLocation3D.x, event.startLocation3D.y, event.startLocation3D.z, 1.0)
-            let transformedPoint = transformHolder.transform.matrix * localPoint
-            let globalPoint3D = Point3D(x: transformedPoint.x, y: transformedPoint.y, z: transformedPoint.z)
-
+        if spatializedElement.enableDragStartGesture, !gestureState.isDrag {
+            let frameZ = localFrameOffsetZ()
+            let startLocal = Point3D(
+                x: event.startLocation3D.x,
+                y: event.startLocation3D.y,
+                z: event.startLocation3D.z - frameZ
+            )
+            let globalPoint3D = localToScene(event.startLocation3D)
             let gestureEvent = WebSpatialDragStartGuestureEvent(detail: .init(
-                startLocation3D: event.startLocation3D,
+                startLocation3D: startLocal,
                 globalLocation3D: globalPoint3D
             ))
-
             spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
         }
 
@@ -88,24 +97,40 @@ struct SpatializedElementView<Content: View>: View {
             spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
         }
 
-        gestureFlags.isDrag = true
+        gestureState.isDrag = true
     }
 
     private func onDraggingEnded(_ event: DragGesture.Value) {
-        gestureFlags.isDrag = false
+        gestureState.isDrag = false
         if spatializedElement.enableDragEndGesture {
             let gestureEvent = WebSpatialDragEndGuestureEvent()
             spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
         }
     }
 
+    /// Z offset that defines the local coordinate system (front face = z=0).
+    /// Only backOffset and zIndex*bias; excludes translation.z (CSS translateZ is visual-only).
+    private func localFrameOffsetZ() -> Double {
+        (spatializedElement.zIndex * zOrderBias) + spatializedElement.backOffset
+    }
+
+    /// Maps a point in the gesture's local coordinate system to SpatialScene.
+    private func localToScene(_ localPoint: Point3D) -> Point3D {
+        let p = SIMD4<Double>(localPoint.x, localPoint.y, localPoint.z, 1.0)
+        let scene = gestureState.proxyTransform.matrix * p
+        return Point3D(x: scene.x, y: scene.y, z: scene.z)
+    }
+
     private func onTapEnded(_ event: SpatialTapGesture.Value) {
         if spatializedElement.enableTapGesture {
-            let localPoint = SIMD4<Double>(event.location3D.x, event.location3D.y, event.location3D.z, 1.0)
-            let transformedPoint = transformHolder.transform.matrix * localPoint
-            let globalPoint3D = Point3D(x: transformedPoint.x, y: transformedPoint.y, z: transformedPoint.z)
-
-            spatialScene.sendWebMsg(spatializedElement.id, WebSpatialTapGuestureEvent(detail: .init(location3D: event.location3D, globalLocation3D: globalPoint3D)))
+            let frameZ = localFrameOffsetZ()
+            let localPoint3D = Point3D(
+                x: event.location3D.x,
+                y: event.location3D.y,
+                z: event.location3D.z - frameZ
+            )
+            let globalPoint3D = localToScene(event.location3D)
+            spatialScene.sendWebMsg(spatializedElement.id, WebSpatialTapGuestureEvent(detail: .init(location3D: localPoint3D, globalLocation3D: globalPoint3D)))
         }
     }
 
@@ -130,9 +155,6 @@ struct SpatializedElementView<Content: View>: View {
 
     var body: some View {
         let transform = spatializedElement.transform
-        let translation = transform.translation
-        let scale = transform.scale
-        let rotation = transform.rotation!
 
         let width = spatializedElement.width
         let height = spatializedElement.height
@@ -146,8 +168,17 @@ struct SpatializedElementView<Content: View>: View {
         let visible = spatializedElement.visible
         let enableGesture = spatializedElement.enableGesture
 
-        let z = translation.z + (spatializedElement.zIndex * zOrderBias)
-        let smallOffset = z == 0.0 ? 0.0001 : 0
+        let frameOffsetZ = localFrameOffsetZ()
+        let smallOffset = abs(frameOffsetZ) < 0.0001 ? 0.0001 : 0
+
+        // Wrap CSS transform with anchor (CSS transform-origin) since
+        // transform3DEffect does not support anchor. Preserves the original
+        // CSS transform order (e.g. rotateX(90deg) translateZ(100px)).
+        let ax = width * anchor.x
+        let ay = height * anchor.y
+        let toAnchor = AffineTransform3D(translation: Vector3D(x: -ax, y: -ay, z: 0))
+        let fromAnchor = AffineTransform3D(translation: Vector3D(x: ax, y: ay, z: 0))
+        let anchoredTransform = fromAnchor.concatenating(transform).concatenating(toAnchor)
 
         // when spatialdiv have regular/thick/thin material and alignment is back, there'll be a bug that clipping content
         // so when spatializedElement is spatialdiv, .center alignment will be applied
@@ -156,34 +187,26 @@ struct SpatializedElementView<Content: View>: View {
         content
             .frame(width: width, height: height)
             .frame(depth: depth, alignment: alignment)
-            .onGeometryChange3D(for: AffineTransform3D.self) { proxy in
-                let rect3d = proxy.frame(in: .named("SpatialScene"))
-                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerClientCube(origin: rect3d.origin, size: rect3d.size))
-                let transform = proxy.transform(in: .named("SpatialScene"))!
-                transformHolder.transform = transform
-                return transform
-            } action: { _, new in
-                spatialScene.sendWebMsg(spatializedElement.id, SpatiaizedContainerTransform(detail: new))
-            }
             .frame(depth: 0, alignment: .back)
             // use .offset(smallVal) to workaround for glassEffect not working and small width/height spatialDiv not working
             .offset(z: smallOffset)
-            .scaleEffect(
-                x: scale.width,
-                y: scale.height,
-                z: scale.depth,
-                anchor: anchor
-            )
-            .rotation3DEffect(
-                rotation,
-                anchor: anchor
-            )
-            .offset(x: translation.x, y: translation.y)
-            .offset(z: z)
+            // Full CSS transform matrix with anchor baked in. Preserves transform
+            // composition order; CSS translateZ participates in rotation direction.
+            .transform3DEffect(anchoredTransform)
+            // backOffset + zIndex: always along parent Z, independent of CSS transform.
+            .offset(z: frameOffsetZ)
+            // Gesture before .position(): event.location3D is in the element's local space
+            // (top-left origin), and does not include visual transforms.
+            .simultaneousGesture(enableGesture ? gesture : nil)
+            .onGeometryChange3D(for: AffineTransform3D.self) { proxy in
+                proxy.transform(in: .named("SpatialScene"))!
+            } action: { new in
+                gestureState.proxyTransform = new
+                spatializedElement.proxySceneTransform = new
+            }
+
             .position(x: centerX + width / 2, y: centerY + height / 2)
-            .offset(z: spatializedElement.backOffset)
             .opacity(opacity)
             .hidden(!visible)
-            .simultaneousGesture(enableGesture ? gesture : nil)
     }
 }

package/web-spatial/view/SpatializedStatic3DView.swift

@@ -28,7 +28,11 @@ struct SpatializedStatic3DView: View {
         let z = translation.z
 
         let enableGesture = spatializedElement.enableGesture
-        if let url = URL(string: spatializedStatic3DElement.modelURL) {
+        let rawModelURL = spatializedStatic3DElement.modelURL
+        let modelURL = rawModelURL.hasPrefix("file://")
+            ? pwaManager.getLocalResourceURL(url: rawModelURL)
+            : rawModelURL
+        if let url = URL(string: modelURL) {
             Model3D(url: url) { newPhase in
                 switch newPhase {
                 case .empty:

package/web-spatial/webview/SpatialWebController.swift

@@ -113,7 +113,21 @@ class SpatialWebController: NSObject, WKNavigationDelegate, WKScriptMessageHandl
         print("urlSchemeTask")
         let url = urlSchemeTask.request.url
         if url!.absoluteString.starts(with: "file://") {
-            let urlRequest = urlSchemeTask.request
+            let resource: String = pwaManager.getLocalResourceURL(url: url!.absoluteString)
+            var urlRequest = urlSchemeTask.request
+            if resource != "" {
+                if let resourceUrl = URL(string: resource) {
+                    urlRequest = URLRequest(url: resourceUrl)
+                } else {
+                    let error = NSError(domain: "LocalResourceError", code: 404, userInfo: [NSLocalizedDescriptionKey: "Local file resource mapping failed"])
+                    urlSchemeTask.didFailWithError(error)
+                    return
+                }
+            } else {
+                let error = NSError(domain: "LocalResourceError", code: 404, userInfo: [NSLocalizedDescriptionKey: "Local resource not found"])
+                urlSchemeTask.didFailWithError(error)
+                return
+            }
 
             let session = URLSession(configuration: URLSessionConfiguration.default)
             let dataTask = session.dataTask(with: urlRequest) { [task = urlSchemeTask as AnyObject] data, response, _ in

package/web-spatial.xcodeproj/project.pbxproj

@@ -155,22 +155,22 @@
 /* End PBXFrameworksBuildPhase section */
 
 /* Begin PBXGroup section */
-		2B2F1D5A2BEBFAAA006897EE = {
+		2B1008A32F0B000800000003 /* web-spatialTests */ = {
 			isa = PBXGroup;
 			children = (
-				2B2F1D652BEBFAAA006897EE /* web-spatial */,
-				2B1008A32F0B000800000003 /* web-spatialTests */,
-				2B2F1D662BEBFAAA006897EE /* Packages */,
-				2B2F1D642BEBFAAA006897EE /* Products */,
+				2B1008A12F0B000800000001 /* NavigationCleanupTests.swift */,
 			);
+			path = "web-spatialTests";
 			sourceTree = "<group>";
 		};
-		2B1008A32F0B000800000003 /* web-spatialTests */ = {
+		2B2F1D5A2BEBFAAA006897EE = {
 			isa = PBXGroup;
 			children = (
-				2B1008A12F0B000800000001 /* NavigationCleanupTests.swift */,
+				2B2F1D652BEBFAAA006897EE /* web-spatial */,
+				2B1008A32F0B000800000003 /* web-spatialTests */,
+				2B2F1D662BEBFAAA006897EE /* Packages */,
+				2B2F1D642BEBFAAA006897EE /* Products */,
 			);
-			path = "web-spatialTests";
 			sourceTree = "<group>";
 		};
 		2B2F1D642BEBFAAA006897EE /* Products */ = {