@mmerterden/multi-agent-pipeline 20.7.0 → 20.8.0

package/pipeline/skills/shared/external/realitykit-ar/references/realitykit-patterns.md

@@ -1,480 +1,353 @@
-# RealityKit + ARKit Extended Patterns
+# RealityKit Patterns
 
-Overflow reference for the `realitykit` skill. Contains advanced patterns that exceed the main skill file's scope.
+Longer examples for the `realitykit-ar` skill. SKILL.md covers setup, anchoring,
+raycasting and gestures; this file goes deeper.
 
 ## Contents
 
-- [Physics Simulation](#physics-simulation)
-- [Entity Animations](#entity-animations)
-- [Custom Materials and Lighting](#custom-materials-and-lighting)
-- [Entity Component System](#entity-component-system)
-- [Occlusion and Environment](#occlusion-and-environment)
-- [RealityKit + SwiftUI Integration](#realitykit--swiftui-integration)
-- [Performance Tips](#performance-tips)
+- [Physics](#physics)
+- [Animation](#animation)
+- [Materials and lighting](#materials-and-lighting)
+- [Custom components and systems](#custom-components-and-systems)
+- [Occlusion and environment](#occlusion-and-environment)
+- [A SwiftUI model viewer](#a-swiftui-model-viewer)
+- [Entity accessibility](#entity-accessibility)
+- [Performance](#performance)
 
-## Physics Simulation
+## Physics
 
-### Rigid Body Physics
+A static floor and a bouncing ball:
 
 ```swift
-import RealityKit
-
-func createPhysicsScene(content: RealityViewCameraContent) {
-    // Ground plane (static body)
-    let ground = ModelEntity(
-        mesh: .generatePlane(width: 2, depth: 2),
-        materials: [SimpleMaterial(color: .gray, isMetallic: false)]
-    )
-    ground.components.set(PhysicsBodyComponent(
-        massProperties: .default,
-        material: .default,
-        mode: .static
-    ))
-    ground.components.set(CollisionComponent(
-        shapes: [.generateBox(size: [2, 0.01, 2])]
-    ))
-    ground.position = [0, -0.5, -1]
-    content.add(ground)
-
-    // Falling box (dynamic body)
-    let box = ModelEntity(
-        mesh: .generateBox(size: 0.1),
-        materials: [SimpleMaterial(color: .red, isMetallic: true)]
-    )
-    box.components.set(PhysicsBodyComponent(
-        massProperties: .init(mass: 1.0),
-        material: PhysicsMaterialResource.generate(
-            staticFriction: 0.5,
-            dynamicFriction: 0.5,
-            restitution: 0.7
-        ),
-        mode: .dynamic
-    ))
-    box.components.set(CollisionComponent(
-        shapes: [.generateBox(size: [0.1, 0.1, 0.1])]
-    ))
-    box.position = [0, 0.5, -1]
-    content.add(box)
+@MainActor
+func makeCourt() -> (floor: ModelEntity, ball: ModelEntity) {
+    let floor = ModelEntity(mesh: .generatePlane(width: 3, depth: 3))
+    floor.components.set(PhysicsBodyComponent(massProperties: .default, material: .default, mode: .static))
+    floor.components.set(CollisionComponent(shapes: [.generateBox(size: [3, 0.005, 3])]))
+
+    let bounce = PhysicsMaterialResource.generate(staticFriction: 0.4, dynamicFriction: 0.3, restitution: 0.8)
+    let ball = ModelEntity(mesh: .generateSphere(radius: 0.05))
+    ball.components.set(PhysicsBodyComponent(massProperties: .init(mass: 0.6), material: bounce, mode: .dynamic))
+    ball.components.set(CollisionComponent(shapes: [.generateSphere(radius: 0.05)]))
+    ball.position = [0, 1, 0]
+    return (floor, ball)
 }
 ```
 
-### Applying Forces and Impulses
+Entities with a physics body conform to `HasPhysicsBody`, which adds:
 
-```swift
-// Apply a continuous force (in Newtons)
-if var physics = entity.components[PhysicsBodyComponent.self] {
-    // Forces are applied per-frame via the physics simulation
-}
+- `addForce(_:relativeTo:)`: a continuous force in newtons, applied each frame
+  you call it.
+- `applyLinearImpulse(_:relativeTo:)`: an instant kick.
 
-// Apply an instantaneous impulse
-if let physicsEntity = entity as? HasPhysicsBody {
-    physicsEntity.addForce([0, 10, 0], relativeTo: nil)
-    physicsEntity.applyLinearImpulse([5, 0, 0], relativeTo: nil)
+```swift
+@MainActor
+func kick(_ ball: ModelEntity) {
+    ball.applyLinearImpulse([0.3, 1.2, 0], relativeTo: nil)
 }
 ```
 
-### Collision Detection
+Collision events:
 
 ```swift
 RealityView { content in
-    // Setup entities...
-
-    _ = content.subscribe(to: CollisionEvents.Began.self) { event in
-        print("Collision between \(event.entityA.name) and \(event.entityB.name)")
+    let court = makeCourt()
+    content.add(court.floor)
+    content.add(court.ball)
+    _ = content.subscribe(to: CollisionEvents.Began.self) { contact in
+        Log.scene.debug("\(contact.entityA.name) touched \(contact.entityB.name)")
     }
-
-    _ = content.subscribe(to: CollisionEvents.Ended.self) { event in
-        print("Collision ended between \(event.entityA.name) and \(event.entityB.name)")
+    _ = content.subscribe(to: CollisionEvents.Ended.self) { contact in
+        Log.scene.debug("\(contact.entityA.name) separated")
     }
 }
 ```
 
-## Entity Animations
+## Animation
 
-### Transform Animation
+Move an entity by animating to a changed copy of its transform:
 
 ```swift
-func animateEntity(_ entity: Entity) {
-    // Move to a new position over 2 seconds
-    var transform = entity.transform
-    transform.translation = [0.5, 0, -0.5]
-
-    entity.move(
-        to: transform,
-        relativeTo: entity.parent,
-        duration: 2.0,
-        timingFunction: .easeInOut
-    )
+@MainActor
+func slideUp(_ panel: Entity) {
+    var target = panel.transform
+    target.translation.y += 0.2
+    panel.move(to: target, relativeTo: panel.parent, duration: 0.8, timingFunction: .easeInOut)
 }
 ```
 
-### Orbit Animation
+Circle an entity around a point with a per-frame subscription:
 
 ```swift
-func orbitAnimation(
-    entity: Entity,
-    content: RealityViewCameraContent
-) {
+RealityView { content in
+    let moon = ModelEntity(mesh: .generateSphere(radius: 0.02))
+    content.add(moon)
+    let center: SIMD3<Float> = [0, 0, -0.7]
     var angle: Float = 0
-    let radius: Float = 0.3
-    let center = SIMD3<Float>(0, 0, -0.8)
-
-    _ = content.subscribe(to: SceneEvents.Update.self) { event in
-        angle += Float(event.deltaTime) * 1.5
-        entity.position = center + SIMD3<Float>(
-            cos(angle) * radius,
-            0,
-            sin(angle) * radius
-        )
+    _ = content.subscribe(to: SceneEvents.Update.self) { frame in
+        angle += Float(frame.deltaTime) * 0.9
+        moon.position = center + [0.25 * cos(angle), 0, 0.25 * sin(angle)]
     }
 }
 ```
 
-### Playing USDZ Animations
+Play an animation baked into a USDZ file:
 
 ```swift
-RealityView { content in
-    if let character = try? await ModelEntity(named: "character") {
-        content.add(character)
-
-        // Play all available animations
-        if let animation = character.availableAnimations.first {
-            character.playAnimation(
-                animation.repeat(duration: .infinity),
-                transitionDuration: 0.5,
-                startsPaused: false
-            )
-        }
-    }
+@MainActor
+func startIdleLoop(on robot: Entity) {
+    guard let clip = robot.availableAnimations.first else { return }
+    robot.playAnimation(clip.repeat(duration: .infinity), transitionDuration: 0.4, startsPaused: false)
 }
 ```
 
-## Custom Materials and Lighting
-
-### SimpleMaterial Variations
+## Materials and lighting
 
 ```swift
-// Metallic material
-let metallic = SimpleMaterial(
-    color: .init(tint: .gray, texture: nil),
-    roughness: .float(0.1),
-    isMetallic: true
-)
-
-// Transparent material
-var transparent = SimpleMaterial()
-transparent.color = .init(
-    tint: UIColor.blue.withAlphaComponent(0.5),
-    texture: nil
-)
-transparent.blending = .transparent(opacity: 0.5)
-
-// Textured material
-if let texture = try? await TextureResource(named: "wood") {
-    var textured = SimpleMaterial()
-    textured.color = .init(
-        tint: .white,
-        texture: .init(texture)
-    )
+@MainActor
+func chrome() -> SimpleMaterial {
+    SimpleMaterial(color: .lightGray, roughness: .float(0.05), isMetallic: true)
 }
-```
 
-### Environment Lighting
+@MainActor
+func frostedGlass() -> PhysicallyBasedMaterial {
+    var glass = PhysicallyBasedMaterial()
+    glass.baseColor = .init(tint: .white)
+    glass.roughness = 0.3
+    glass.blending = .transparent(opacity: 0.5)
+    return glass
+}
 
-```swift
-RealityView { content in
-    // Use image-based lighting
-    if let resource = try? await EnvironmentResource(named: "studio") {
-        content.environment.lighting.resource = resource
-        content.environment.lighting.intensityExponent = 1.0
+@MainActor
+func woodGrain() async -> SimpleMaterial {
+    var wood = SimpleMaterial()
+    if let grain = try? await TextureResource(named: "oak_diffuse") {
+        wood.color = .init(tint: .white, texture: .init(grain))
     }
+    return wood
 }
 ```
 
-## Entity Component System
-
-### Custom Components
+`SimpleMaterial` has no `blending` property; transparency needs
+`PhysicallyBasedMaterial` or `UnlitMaterial`.
 
-Define custom components to attach data to entities:
+Image-based lighting from an environment resource: put an
+`ImageBasedLightComponent` on one entity and point each lit entity at it with
+`ImageBasedLightReceiverComponent`. `RealityViewCameraContent.environment` has
+no lighting properties on iOS.
 
 ```swift
-import RealityKit
-
-struct HealthComponent: Component {
-    var currentHealth: Float
-    var maxHealth: Float
-
-    var healthPercentage: Float {
-        currentHealth / maxHealth
-    }
+RealityView { content in
+    guard let studio = try? await EnvironmentResource(named: "softbox") else { return }
+    let light = Entity()
+    light.components.set(ImageBasedLightComponent(source: .single(studio), intensityExponent: 1.2))
+    content.add(light)
+
+    let vase = ModelEntity(mesh: .generateSphere(radius: 0.1), materials: [chrome()])
+    vase.components.set(ImageBasedLightReceiverComponent(imageBasedLight: light))
+    content.add(vase)
 }
-
-// Register the component (once, at app launch)
-HealthComponent.registerComponent()
-
-// Attach to an entity
-let enemy = ModelEntity(
-    mesh: .generateBox(size: 0.1),
-    materials: [SimpleMaterial(color: .red, isMetallic: false)]
-)
-enemy.components.set(HealthComponent(
-    currentHealth: 100,
-    maxHealth: 100
-))
 ```
 
-### Custom Systems
-
-Systems process entities with matching components each frame:
+## Custom components and systems
 
 ```swift
-struct HealthColorSystem: System {
-    static let query = EntityQuery(where: .has(HealthComponent.self))
+struct Hover: Component {
+    var height: Float = 0.03
+    var speed: Float = 2
+    var phase: Float = 0
+    var offset: Float { sin(phase) * height }
+}
 
-    init(scene: Scene) { }
+struct HoverSystem: System {
+    static let query = EntityQuery(where: .has(Hover.self))
+
+    init(scene: RealityKit.Scene) {}
 
     func update(context: SceneUpdateContext) {
-        for entity in context.entities(
-            matching: Self.query,
-            updatingSystemWhen: .rendering
-        ) {
-            guard let health = entity.components[HealthComponent.self],
-                  var model = entity.components[ModelComponent.self]
-            else { continue }
-
-            let ratio = health.healthPercentage
-            let color = UIColor(
-                red: CGFloat(1 - ratio),
-                green: CGFloat(ratio),
-                blue: 0,
-                alpha: 1
-            )
-            model.materials = [SimpleMaterial(
-                color: color,
-                isMetallic: false
-            )]
-            entity.components.set(model)
+        for entity in context.entities(matching: Self.query, updatingSystemWhen: .rendering) {
+            guard var hover = entity.components[Hover.self] else { continue }
+            hover.phase += hover.speed * Float(context.deltaTime)
+            entity.position.y = hover.offset
+            entity.components[Hover.self] = hover
+
+            if var model = entity.components[ModelComponent.self], hover.offset > 0 {
+                model.materials = [SimpleMaterial(color: .systemYellow, isMetallic: false)]
+                entity.components[ModelComponent.self] = model
+            }
         }
     }
 }
+```
+
+Register both once at launch, then attach the component to entities:
 
-// Register the system
-HealthColorSystem.registerSystem()
+```swift
+Hover.registerComponent()
+HoverSystem.registerSystem()
+lantern.components.set(Hover())
 ```
 
-## Occlusion and Environment
+Components are value types, so read, change and write back; the same goes for
+`ModelComponent`.
 
-### Occlusion Material
+## Occlusion and environment
 
-Use occlusion materials to hide virtual content behind real-world surfaces:
+An invisible occluder hides virtual objects behind a real table or wall:
 
 ```swift
-let occlusionPlane = ModelEntity(
-    mesh: .generatePlane(width: 1, depth: 1),
-    materials: [OcclusionMaterial()]
-)
-occlusionPlane.position = [0, -0.5, -1]
-content.add(occlusionPlane)
+let tableTop = ModelEntity(mesh: .generatePlane(width: 1.2, depth: 0.7), materials: [OcclusionMaterial()])
 ```
 
-This makes virtual objects appear to go behind real-world surfaces.
-
-### Environment Configuration
+In AR mode the camera feed is the background. With the `.virtual` camera, the
+iOS `RealityViewEnvironment` offers `.default` and `.skybox(_:)`:
 
 ```swift
-RealityView { content in
-    // Configure the background
-    content.environment.background = .color(.clear)  // AR camera passthrough
-
-    // Adjust lighting intensity
-    content.environment.lighting.intensityExponent = 1.2
+content.camera = .virtual
+if let sky = try? await EnvironmentResource(named: "softbox") {
+    content.environment = .skybox(sky)
 }
 ```
 
-## RealityKit + SwiftUI Integration
-
-### AR Model Viewer with Controls
+## A SwiftUI model viewer
 
 ```swift
-import SwiftUI
-import RealityKit
 import AVFoundation
+import Observation
+import RealityKit
+import SwiftUI
 
-@Observable
-@MainActor
-final class ARViewModel {
-    var selectedModelName: String = "robot"
-    var modelScale: Float = 0.01
-    var isPlaced = false
-
-    private var currentEntity: ModelEntity?
-
-    func loadModel(into content: RealityViewCameraContent) async {
-        // Remove existing model
-        currentEntity?.removeFromParent()
-
-        guard let model = try? await ModelEntity(
-            named: selectedModelName
-        ) else { return }
-
-        model.scale = [modelScale, modelScale, modelScale]
-        model.position = [0, -0.2, -0.8]
-        model.name = "currentModel"
-
-        // Enable interaction
-        model.generateCollisionShapes(recursive: true)
-        model.components.set(InputTargetComponent())
-
-        content.add(model)
-        currentEntity = model
-        isPlaced = true
+@Observable @MainActor
+final class ShowroomModel {
+    var selection = "chair"
+    var scale: Float = 0.01
+    private var current: ModelEntity?
+
+    func show(in content: RealityViewCameraContent) async {
+        current?.removeFromParent()
+        guard let piece = try? await ModelEntity(named: selection) else { return }
+        piece.name = "showpiece"
+        piece.scale = SIMD3(repeating: scale)
+        piece.position = [0, -0.1, -0.6]
+        piece.generateCollisionShapes(recursive: true)
+        piece.components.set(InputTargetComponent())
+        content.add(piece)
+        current = piece
     }
 }
 
-struct ARModelViewer: View {
-    @State private var viewModel = ARViewModel()
-    @State private var cameraAuthorized = false
+struct ShowroomView: View {
+    @State private var model = ShowroomModel()
+    @State private var cameraAllowed = false
 
     var body: some View {
-        ZStack {
-            if cameraAuthorized {
+        Group {
+            if cameraAllowed {
                 RealityView { content in
-                    await viewModel.loadModel(into: content)
+                    await model.show(in: content)
                 } update: { content in
-                    if let model = content.entities.first(
-                        where: { $0.name == "currentModel" }
-                    ) {
-                        let s = viewModel.modelScale
-                        model.scale = [s, s, s]
-                    }
+                    content.entities.first { $0.name == "showpiece" }?.scale = SIMD3(repeating: model.scale)
                 }
                 .gesture(
                     DragGesture()
                         .targetedToAnyEntity()
-                        .onChanged { value in
-                            value.entity.position = value.convert(
-                                value.location3D,
-                                from: .local,
-                                to: value.entity.parent!
-                            )
+                        .onChanged { drag in
+                            guard let parent = drag.entity.parent,
+                                  let point = drag.unproject(\.location, to: parent) else { return }
+                            drag.entity.position = point
                         }
                 )
+                .overlay(alignment: .bottom) { controls }
             } else {
-                ContentUnavailableView(
-                    "Camera Required",
-                    systemImage: "camera.fill",
-                    description: Text("Allow camera access for AR.")
-                )
-            }
-
-            VStack {
-                Spacer()
-                controlsPanel
-            }
-        }
-        .task {
-            let status = AVCaptureDevice.authorizationStatus(for: .video)
-            if status == .authorized {
-                cameraAuthorized = true
-            } else if status == .notDetermined {
-                cameraAuthorized = await AVCaptureDevice
-                    .requestAccess(for: .video)
+                ContentUnavailableView("Camera needed", systemImage: "camera",
+                                       description: Text("Allow camera access in Settings to place furniture."))
             }
         }
+        .task { cameraAllowed = await cameraReady() }
     }
 
-    private var controlsPanel: some View {
+    private var controls: some View {
         VStack {
-            Picker("Model", selection: $viewModel.selectedModelName) {
-                Text("Robot").tag("robot")
-                Text("Chair").tag("chair")
-                Text("Lamp").tag("lamp")
+            Picker("Item", selection: $model.selection) {
+                Text("Armchair").tag("chair")
+                Text("Desk lamp").tag("desk_lamp")
             }
             .pickerStyle(.segmented)
-
-            Slider(
-                value: $viewModel.modelScale,
-                in: 0.001...0.05,
-                step: 0.001
-            ) {
-                Text("Scale")
-            }
+            Slider(value: $model.scale, in: 0.001...0.05, step: 0.001)
         }
         .padding()
-        .background(.ultraThinMaterial)
-        .clipShape(.rect(cornerRadius: 16))
+        .background(.ultraThinMaterial, in: .rect(cornerRadius: 20))
         .padding()
     }
 }
 ```
 
-### Entity Accessibility
+`cameraReady()` is the permission helper from SKILL.md. The drag handler
+guards for a parent entity instead of assuming one exists.
+`generateCollisionShapes(recursive: true)` together with
+`InputTargetComponent()` makes a loaded model interactive.
 
-Make AR content accessible to assistive technologies:
+## Entity accessibility
 
 ```swift
-let model = ModelEntity(
-    mesh: .generateBox(size: 0.1),
-    materials: [SimpleMaterial(color: .blue, isMetallic: false)]
-)
-var accessibility = AccessibilityComponent()
-accessibility.isAccessibilityElement = true
-accessibility.label = "Blue cube"
-accessibility.value = "Interactive 3D object"
-accessibility.traits = [.button]
-accessibility.systemActions = [.activate]
-model.components.set(accessibility)
+@MainActor
+func describe(_ door: Entity) {
+    var info = AccessibilityComponent()
+    info.isAccessibilityElement = true
+    info.label = "Garage door"
+    info.value = "Closed"
+    info.traits = [.button]
+    info.systemActions = [.activate]
+    door.components.set(info)
+}
 ```
 
-## Performance Tips
-
-### Entity Pooling
+## Performance
 
-Reuse entities instead of creating and destroying them repeatedly:
+Keep a pool of entities and recycle them, rather than allocating fresh ones
+and throwing them away:
 
 ```swift
 @MainActor
-final class EntityPool {
-    private var available: [ModelEntity] = []
-    private let mesh: MeshResource
-    private let materials: [any Material]
-
-    init(mesh: MeshResource, materials: [any Material], initialCount: Int) {
-        self.mesh = mesh
-        self.materials = materials
-        for _ in 0..<initialCount {
-            let entity = ModelEntity(mesh: mesh, materials: materials)
-            entity.isEnabled = false
-            available.append(entity)
+final class DebrisPool {
+    private var idle: [ModelEntity] = []
+    private let mesh = MeshResource.generateBox(size: 0.02)
+    private let paint = [SimpleMaterial(color: .gray, isMetallic: false)]
+
+    init(size: Int) {
+        for _ in 0..<size {
+            let chip = ModelEntity(mesh: mesh, materials: paint)
+            chip.isEnabled = false
+            idle.append(chip)
         }
     }
 
-    func acquire() -> ModelEntity {
-        if let entity = available.popLast() {
-            entity.isEnabled = true
-            return entity
-        }
-        return ModelEntity(mesh: mesh, materials: materials)
+    func take() -> ModelEntity {
+        let chip = idle.popLast() ?? ModelEntity(mesh: mesh, materials: paint)
+        chip.isEnabled = true
+        return chip
     }
 
-    func release(_ entity: ModelEntity) {
-        entity.isEnabled = false
-        entity.removeFromParent()
-        available.append(entity)
+    func giveBack(_ chip: ModelEntity) {
+        chip.isEnabled = false
+        chip.removeFromParent()
+        idle.append(chip)
     }
 }
 ```
 
-### Reducing Draw Calls
+Fewer draw calls:
 
-- Merge static entities into a single entity when possible
-- Use instancing for repeated identical geometry
-- Limit the number of unique materials in a scene
-- Use LOD (level of detail) models for objects at varying distances
+- Merge static entities where you can.
+- Use instancing for geometry that repeats.
+- Keep the number of distinct materials low.
+- Swap in lower-detail models as distance grows.
 
-### Memory Management
+Memory:
 
-- Unload entities that are no longer visible
-- Use `ModelEntity(named:in:)` with a specific bundle to control resource loading
-- Monitor memory with Instruments (RealityKit Trace template)
-- Set `entity.isEnabled = false` instead of removing entities you will reuse
+- Unload entities nobody can see.
+- Load with `ModelEntity(named:in:)` and an explicit bundle to control where
+  resources come from.
+- Profile with the RealityKit Trace template in Instruments.
+- For entities you will show again, set `isEnabled = false` rather than removing
+  them.