@webspatial/platform-visionos 1.5.0 → 1.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webspatial/platform-visionos",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Used to publish WebSpatial projects to Apple Vision Pro",
   "type": "commonjs",
   "main": "package.json",

package/web-spatial/JSBCommand.swift

@@ -14,7 +14,8 @@ struct AddSpatializedElementToSpatialScene: CommandDataProtocol {
 
 struct CreateSpatializedStatic3DElement: CommandDataProtocol {
     static let commandType: String = "CreateSpatializedStatic3DElement"
-    let modelURL: String
+    let modelURL: String?
+    let sources: [ModelSource]?
 }
 
 struct CreateSpatializedDynamic3DElement: CommandDataProtocol {
@@ -260,7 +261,12 @@ struct UpdateSpatializedStatic3DElementProperties: SpatializedElementProperties
     let rotateConstrainedToAxis: Vec3?
 
     let modelURL: String?
+    let sources: [ModelSource]?
     let modelTransform: [Double]?
+    let autoplay: Bool?
+    let loop: Bool?
+    let animationPaused: Bool?
+    let playbackRate: Double?
 }
 
 struct UpdateSpatializedDynamic3DElementProperties: SpatializedElementProperties {

package/web-spatial/WebMsgCommand.swift

@@ -25,6 +25,8 @@ enum SpatialWebMsgType: String, Encodable {
     case spatialmagnify
     case spatialmagnifyend
 
+    case animationstatechange
+
     case objectdestroy
 }
 
@@ -97,14 +99,33 @@ struct WebSpatialMagnifyEndGuestureEvent: Encodable {
     let type: SpatialWebMsgType = .spatialmagnifyend
 }
 
+struct ModelLoadSuccessDetail: Encodable {
+    let src: String
+}
+
 struct ModelLoadSuccess: Encodable {
     let type: SpatialWebMsgType = .modelloaded
+    let detail: ModelLoadSuccessDetail
+
+    init(src: String) {
+        detail = ModelLoadSuccessDetail(src: src)
+    }
 }
 
 struct ModelLoadFailure: Encodable {
     let type: SpatialWebMsgType = .modelloadfailed
 }
 
+struct AnimationStateChangeDetail: Encodable {
+    let paused: Bool
+    let duration: Double
+}
+
+struct AnimationStateChangeEvent: Encodable {
+    let type: SpatialWebMsgType = .animationstatechange
+    let detail: AnimationStateChangeDetail
+}
+
 struct SpatialObjectDestroiedEvent: Encodable {
     let type: SpatialWebMsgType = .objectdestroy
 }

package/web-spatial/model/SpatialScene.swift

@@ -85,6 +85,8 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
 
     // TOPIC end
 
+    var isSpatialElementGestureActive = false
+
     var spatialWebViewModel: SpatialWebViewModel
 
     private var meterToPtUnscaled: Double?
@@ -429,7 +431,7 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         }
     }
 
-    // Temporary storage for webview models awaiting JSB initialization
+    /// Temporary storage for webview models awaiting JSB initialization
     private var pendingAttachmentWebViewModels = [String: SpatialWebViewModel]()
 
     private func handleCreateAttachment(_ url: URL) -> WebViewElementInfo? {
@@ -543,6 +545,9 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
     private func onCreateSpatializedStatic3DElement(command: CreateSpatializedStatic3DElement, resolve: @escaping JSBManager.ResolveHandler<Encodable>) {
         let spatialObject: SpatializedStatic3DElement = createSpatializedElement(.SpatializedStatic3DElement)
         spatialObject.modelURL = command.modelURL
+        if let sources = command.sources {
+            spatialObject.sources = sources
+        }
 
         resolve(.success(AddSpatializedElementReply(id: spatialObject.id)))
     }
@@ -585,6 +590,10 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
 
         updateSpatializedElementProperties(spatializedElement, command)
 
+        if let sources = command.sources {
+            spatializedElement.sources = sources
+        }
+
         if let modelURL = command.modelURL {
             spatializedElement.modelURL = modelURL
         }
@@ -604,6 +613,22 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
             spatializedElement.modelTransform = affineTransform3D
         }
 
+        if let autoplay = command.autoplay {
+            spatializedElement.autoplay = autoplay
+        }
+
+        if let loop = command.loop {
+            spatializedElement.loop = loop
+        }
+
+        if let animationPaused = command.animationPaused {
+            spatializedElement.animationPaused = animationPaused
+        }
+
+        if let playbackRate = command.playbackRate {
+            spatializedElement.playbackRate = playbackRate
+        }
+
         resolve(.success(baseReplyData))
     }
 
@@ -1111,22 +1136,22 @@ class SpatialScene: SpatialObject, ScrollAbleSpatialElementContainer, WebMsgSend
         resolve(.success(ConvertReply(id: command.entityId, position: point)))
     }
 
-    /// Input: command.position, command.fromId, command.toId
-    /// fromId/toId can reference either the scene (window) or an entity.
-    /// Step 1: Convert position to window coordinates (view global, px)
-    ///   - If from is window (scene), position is already in view global (px). Go to Step 2.
-    ///   - If from is 2d frame(SpatializedElement), position is in view local (px).
-    ///     - view local → window (view global, px) using SpatializedElement.convertToScene
-    ///   - If from is an entity, position is in reality entity local (meters):
-    ///     - entity local → reality world (scene)
-    ///     - reality world → window (view global, px)
-    /// Step 2: Convert window coordinates (view global, px) to target output
-    ///   - If to is window, output directly.
-    ///   - If to is 2d frame(SpatializedElement), output in view local (px).
-    ///     - window (view global, px) → view local using SpatializedElement.convertFromScene
-    ///   - If to is an entity, output in reality entity local (meters):
-    ///     - window (view global, px) → reality world (scene)
-    ///     - reality world → reality entity local (meters)
+    // Input: command.position, command.fromId, command.toId
+    // fromId/toId can reference either the scene (window) or an entity.
+    // Step 1: Convert position to window coordinates (view global, px)
+    //   - If from is window (scene), position is already in view global (px). Go to Step 2.
+    //   - If from is 2d frame(SpatializedElement), position is in view local (px).
+    //     - view local → window (view global, px) using SpatializedElement.convertToScene
+    //   - If from is an entity, position is in reality entity local (meters):
+    //     - entity local → reality world (scene)
+    //     - reality world → window (view global, px)
+    // Step 2: Convert window coordinates (view global, px) to target output
+    //   - If to is window, output directly.
+    //   - If to is 2d frame(SpatializedElement), output in view local (px).
+    //     - window (view global, px) → view local using SpatializedElement.convertFromScene
+    //   - If to is an entity, output in reality entity local (meters):
+    //     - window (view global, px) → reality world (scene)
+    //     - reality world → reality entity local (meters)
 
     private func onConvertCoordinate(command: ConvertCoordinate, resolve: @escaping JSBManager.ResolveHandler<Encodable>) {
         func isSceneId(_ id: String) -> Bool {

package/web-spatial/model/SpatializedStatic3DElement.swift

@@ -1,10 +1,25 @@
 import Foundation
 import SwiftUI
 
+struct ModelSource: Codable, Equatable {
+    let src: String
+    let type: String?
+}
+
 @Observable
 class SpatializedStatic3DElement: SpatializedElement {
-    var modelURL: String = ""
+    var modelURL: String?
+    var sources: [ModelSource] = []
     var modelTransform: AffineTransform3D = .identity
+    var autoplay: Bool = false
+    var loop: Bool = false
+    var animationPaused: Bool = true
+    var playbackRate: Double = 1.0
+    var allSources: [ModelSource] {
+        return if let modelURL {
+            [ModelSource(src: modelURL, type: nil)] + sources
+        } else { sources }
+    }
 
     enum CodingKeys: String, CodingKey {
         case modelURL, type

package/web-spatial/view/Spatialized2DElementView.swift

@@ -72,6 +72,9 @@ struct Spatialized2DElementView: View {
         DragGesture()
             .onChanged { gesture in
 //                print("\(spatialized2DElement.name) dragWebGesture")
+                if spatialScene.isSpatialElementGestureActive {
+                    return
+                }
                 if spatialized2DElement.scrollPageEnabled {
                     if !gestureData.dragStarted {
                         gestureData.dragStarted = true
@@ -86,6 +89,9 @@ struct Spatialized2DElementView: View {
             }
             .onEnded { _ in
                 print("\(spatialized2DElement.name) dragWebGestureEnd")
+                if spatialScene.isSpatialElementGestureActive {
+                    return
+                }
                 if spatialized2DElement.scrollPageEnabled {
                     gestureData.dragStarted = false
                     gestureData.dragStart = 0

package/web-spatial/view/SpatializedElementView.swift

@@ -22,7 +22,7 @@ struct SpatializedElementView<Content: View>: View {
 
     /// Begin Interaction
     var gesture: some Gesture {
-        DragGesture(minimumDistance: 10)
+        DragGesture(minimumDistance: 10, coordinateSpace: .named("SpatialScene"))
             .onChanged(onDragging)
             .onEnded(onDraggingEnded)
             .simultaneously(with:
@@ -74,14 +74,13 @@ struct SpatializedElementView<Content: View>: View {
     }
 
     private func onDragging(_ event: DragGesture.Value) {
+        if !gestureState.isDrag {
+            spatialScene.isSpatialElementGestureActive = true
+        }
+
         if spatializedElement.enableDragStartGesture, !gestureState.isDrag {
-            let frameZ = localFrameOffsetZ()
-            let startLocal = Point3D(
-                x: event.startLocation3D.x,
-                y: event.startLocation3D.y,
-                z: event.startLocation3D.z - frameZ
-            )
-            let globalPoint3D = localToScene(event.startLocation3D)
+            let startLocal = sceneToLocal(event.startLocation3D)
+            let globalPoint3D = event.startLocation3D
             let gestureEvent = WebSpatialDragStartGuestureEvent(detail: .init(
                 startLocation3D: startLocal,
                 globalLocation3D: globalPoint3D
@@ -102,6 +101,7 @@ struct SpatializedElementView<Content: View>: View {
 
     private func onDraggingEnded(_ event: DragGesture.Value) {
         gestureState.isDrag = false
+        spatialScene.isSpatialElementGestureActive = false
         if spatializedElement.enableDragEndGesture {
             let gestureEvent = WebSpatialDragEndGuestureEvent()
             spatialScene.sendWebMsg(spatializedElement.id, gestureEvent)
@@ -121,6 +121,11 @@ struct SpatializedElementView<Content: View>: View {
         return Point3D(x: scene.x, y: scene.y, z: scene.z)
     }
 
+    private func sceneToLocal(_ scenePoint: Point3D) -> Point3D {
+        let local = spatializedElement.convertFromScene(SIMD3<Double>(scenePoint.x, scenePoint.y, scenePoint.z))
+        return Point3D(x: local.x, y: local.y, z: local.z)
+    }
+
     private func onTapEnded(_ event: SpatialTapGesture.Value) {
         if spatializedElement.enableTapGesture {
             let frameZ = localFrameOffsetZ()
@@ -198,6 +203,9 @@ struct SpatializedElementView<Content: View>: View {
             // Gesture before .position(): event.location3D is in the element's local space
             // (top-left origin), and does not include visual transforms.
             .simultaneousGesture(enableGesture ? gesture : nil)
+            .onDisappear {
+                spatialScene.isSpatialElementGestureActive = false
+            }
             .onGeometryChange3D(for: AffineTransform3D.self) { proxy in
                 proxy.transform(in: .named("SpatialScene"))!
             } action: { new in

package/web-spatial/view/SpatializedStatic3DView.swift

@@ -5,12 +5,16 @@ struct SpatializedStatic3DView: View {
     @Environment(SpatializedElement.self) var spatializedElement: SpatializedElement
     @Environment(SpatialScene.self) var spatialScene: SpatialScene
 
+    @State private var asset: Model3DAsset?
+    @State private var source: String?
+    @State private var isLoading = false
+
     private var spatializedStatic3DElement: SpatializedStatic3DElement {
         return spatializedElement as! SpatializedStatic3DElement
     }
 
-    func onLoadSuccess() {
-        spatialScene.sendWebMsg(spatializedElement.id, ModelLoadSuccess())
+    func onLoadSuccess(src: String) {
+        spatialScene.sendWebMsg(spatializedElement.id, ModelLoadSuccess(src: src))
     }
 
     func onLoadFailure() {
@@ -28,33 +32,28 @@ struct SpatializedStatic3DView: View {
         let z = translation.z
 
         let enableGesture = spatializedElement.enableGesture
-        let rawModelURL = spatializedStatic3DElement.modelURL
-        let modelURL = rawModelURL.hasPrefix("file://")
-            ? pwaManager.getLocalResourceURL(url: rawModelURL)
-            : rawModelURL
-        if let url = URL(string: modelURL) {
-            Model3D(url: url) { newPhase in
-                switch newPhase {
-                case .empty:
+        if !spatializedStatic3DElement.allSources.isEmpty {
+            Group {
+                if isLoading {
                     ProgressView()
-                case let .success(resolvedModel3D):
-                    resolvedModel3D
-                        .resizable(true)
-                        .aspectRatio(
-                            nil,
-                            contentMode: .fit
-                        )
-                        .if(!depth.isZero) { view in view.scaledToFit3D() }
-                        .onAppear {
-                            self.onLoadSuccess()
-                        }
-                        .if(enableGesture) { view in view.hoverEffect() }
-                case .failure:
+                } else if let asset, let source {
+                    Model3D(asset: asset) { resolvedModel3D in
+                        resolvedModel3D
+                            .resizable(true)
+                            .aspectRatio(
+                                nil,
+                                contentMode: .fit
+                            )
+                            .if(!depth.isZero) { view in view.scaledToFit3D() }
+                            .onAppear {
+                                self.onLoadSuccess(src: source)
+                            }
+                            .if(enableGesture) { view in view.hoverEffect() }
+                    }
+                } else {
                     Text("").onAppear {
                         self.onLoadFailure()
                     }
-                @unknown default:
-                    EmptyView()
                 }
             }
             .scaleEffect(
@@ -67,8 +66,95 @@ struct SpatializedStatic3DView: View {
             )
             .offset(x: x, y: y)
             .offset(z: z)
+            .onChange(of: asset?.animationPlaybackController?.isComplete) { _, isComplete in
+                guard isComplete == true else { return }
+                if spatializedStatic3DElement.loop,
+                   let asset,
+                   let animation = asset.availableAnimations.first
+                {
+                    asset.selectedAnimation = animation
+                    asset.animationPlaybackController?.speed = Float(spatializedStatic3DElement.playbackRate)
+                    asset.animationPlaybackController?.resume()
+                } else {
+                    // Non-looping animation completed naturally; sync paused state to JS
+                    spatializedStatic3DElement.animationPaused = true
+                }
+            }
+            .onChange(of: spatializedStatic3DElement.animationPaused) { onPlayback(isPaused: $1) }
+            .onChange(of: spatializedStatic3DElement.playbackRate) { asset?.animationPlaybackController?.speed = Float($1) }
+            .task(id: spatializedStatic3DElement.allSources) { await loadSources() }
         } else {
             EmptyView()
         }
     }
+
+    /// Plays or pauses the model animation and sends an animation state to the web code
+    private func onPlayback(isPaused: Bool) {
+        guard let asset else {
+            // If entity has not loaded yet and play is called then autoplay after load
+            if !isPaused { spatializedStatic3DElement.autoplay = true }
+            return
+        }
+        // Setting selectedAnimation resets the animation and autoplays on first load
+        if asset.selectedAnimation == nil || asset.animationPlaybackController?.isComplete == true {
+            asset.selectedAnimation = asset.availableAnimations.first
+        }
+        let controller = asset.animationPlaybackController
+        controller?.speed = Float(spatializedStatic3DElement.playbackRate)
+        isPaused ? controller?.pause() : controller?.resume()
+        let duration = controller?.duration ?? 0
+        spatialScene.sendWebMsg(
+            spatializedElement.id,
+            AnimationStateChangeEvent(
+                detail: AnimationStateChangeDetail(paused: isPaused, duration: duration)
+            )
+        )
+    }
+
+    /// Downloads a remote model file and loads it as a Model3DAsset.
+    /// Model3DAsset(url:) requires a local file URL, so remote
+    /// resources must be downloaded first.
+    private func loadAsset(from url: URL) async throws -> Model3DAsset {
+        if url.isFileURL {
+            return try await Model3DAsset(url: url)
+        }
+        let (tempURL, _) = try await URLSession.shared.download(from: url)
+        // TODO: Use FileManager.temporaryDirectory and FileManager.removeItem for auto cleanup
+        let localURL = tempURL.deletingPathExtension()
+            .appendingPathExtension(url.pathExtension)
+        try FileManager.default.moveItem(at: tempURL, to: localURL)
+        return try await Model3DAsset(url: localURL)
+    }
+
+    private func loadSources() async {
+        isLoading = true
+        let result = await loadSources(spatializedStatic3DElement.allSources)
+        asset = result?.asset
+        source = result?.url.absoluteString
+        if spatializedStatic3DElement.autoplay {
+            // If animationPaused didn't change then SwiftUI will not trigger onChange so manually trigger playback
+            // This happens when play is called before load and autoplay is enabled
+            if spatializedStatic3DElement.animationPaused {
+                spatializedStatic3DElement.animationPaused = false
+            } else { onPlayback(isPaused: false) }
+        }
+        isLoading = false
+    }
+
+    /// Attempts to load from each source in order, returning the first success.
+    private func loadSources(_ sources: [ModelSource]) async -> (url: URL, asset: Model3DAsset)? {
+        for source in sources {
+            guard let url = localOrRemoteURL(url: source.src) else { continue }
+            do {
+                return try (url, await loadAsset(from: url))
+            } catch {
+                continue
+            }
+        }
+        return nil
+    }
+}
+
+private func localOrRemoteURL(url: String) -> URL? {
+    URL(string: url.hasPrefix("file://") ? pwaManager.getLocalResourceURL(url: url) : url)
 }