agent-device 0.21.20 → 0.21.21

package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift

@@ -0,0 +1,141 @@
+import AppKit
+import QuartzCore
+
+/// A drawn pointer that shows where a background action lands while the user keeps the real
+/// one. It appears just above and left of the target, glides onto it, and pulses when the action
+/// is delivered: about 0.3 s per action. It holds no state between helper processes.
+final class GhostCursor {
+  private static let size = CGSize(width: 64, height: 64)
+  /// Where the arrow's tip sits inside the panel, from its top-left corner; it leaves room for
+  /// the pulse ring's full radius.
+  private static let tipOffset = CGPoint(x: 22, y: 22)
+  /// Where the glide starts, relative to the target.
+  private static let approach = CGVector(dx: -36, dy: -36)
+
+  private let panel: NSPanel
+  private let view: GhostCursorView
+  private let primaryScreenHeight: CGFloat
+
+  /// Nil when no display is attached; the action still runs, just unseen.
+  static func show() -> GhostCursor? {
+    _ = NSApplication.shared
+    NSApp.setActivationPolicy(.accessory)
+    guard let primary = NSScreen.screens.first else { return nil }
+    return GhostCursor(primaryScreenHeight: primary.frame.height)
+  }
+
+  private init(primaryScreenHeight: CGFloat) {
+    self.primaryScreenHeight = primaryScreenHeight
+    view = GhostCursorView(frame: CGRect(origin: .zero, size: Self.size), tip: Self.tipOffset)
+    panel = NSPanel(
+      contentRect: CGRect(origin: .zero, size: Self.size),
+      styleMask: [.borderless, .nonactivatingPanel],
+      backing: .buffered,
+      defer: false
+    )
+    panel.isOpaque = false
+    panel.backgroundColor = .clear
+    panel.hasShadow = false
+    panel.ignoresMouseEvents = true
+    panel.level = .screenSaver
+    panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary]
+    panel.contentView = view
+  }
+
+  /// Glides onto a point in global top-left coordinates, the space AX and CGEvent share.
+  func move(to target: CGPoint, duration: TimeInterval = 0.15) {
+    let start = CGPoint(x: target.x + Self.approach.dx, y: target.y + Self.approach.dy)
+    place(at: start)
+    panel.orderFrontRegardless()
+    let steps = max(1, Int(duration / 0.016))
+    for step in 1...steps {
+      let t = Double(step) / Double(steps)
+      let eased = 1 - pow(1 - t, 3)
+      place(at: CGPoint(x: start.x + (target.x - start.x) * eased, y: start.y + (target.y - start.y) * eased))
+      flush(for: 0.016)
+    }
+  }
+
+  /// A ring at the tip, marking the moment the action was delivered; nothing when the cursor never
+  /// moved onto a target.
+  func pulse(duration: TimeInterval = 0.12) {
+    guard panel.isVisible else { return }
+    let steps = max(1, Int(duration / 0.016))
+    for step in 1...steps {
+      view.ringProgress = CGFloat(step) / CGFloat(steps)
+      view.display()
+      flush(for: 0.016)
+    }
+  }
+
+  func hide() {
+    panel.orderOut(nil)
+    CATransaction.flush()
+  }
+
+  private func place(at tip: CGPoint) {
+    panel.setFrameOrigin(
+      CGPoint(
+        x: tip.x - Self.tipOffset.x,
+        y: primaryScreenHeight - tip.y - Self.size.height + Self.tipOffset.y
+      )
+    )
+  }
+
+  private func flush(for interval: TimeInterval) {
+    CATransaction.flush()
+    RunLoop.current.run(until: Date(timeIntervalSinceNow: interval))
+  }
+}
+
+private final class GhostCursorView: NSView {
+  private let tip: CGPoint
+  var ringProgress: CGFloat?
+
+  init(frame: CGRect, tip: CGPoint) {
+    self.tip = tip
+    super.init(frame: frame)
+  }
+
+  required init?(coder: NSCoder) {
+    nil
+  }
+
+  override var isFlipped: Bool { true }
+
+  override func draw(_ dirtyRect: NSRect) {
+    NSColor.clear.setFill()
+    dirtyRect.fill()
+    if let progress = ringProgress {
+      let radius = 6 + 14 * progress
+      let ring = NSBezierPath(
+        ovalIn: CGRect(x: tip.x - radius, y: tip.y - radius, width: radius * 2, height: radius * 2)
+      )
+      ring.lineWidth = 3
+      NSColor.systemPurple.withAlphaComponent(0.85 * (1 - progress)).setStroke()
+      ring.stroke()
+    }
+    let arrow = NSBezierPath()
+    arrow.move(to: tip)
+    arrow.line(to: CGPoint(x: tip.x, y: tip.y + 22))
+    arrow.line(to: CGPoint(x: tip.x + 6, y: tip.y + 17))
+    arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 26))
+    arrow.line(to: CGPoint(x: tip.x + 14, y: tip.y + 24))
+    arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 15))
+    arrow.line(to: CGPoint(x: tip.x + 17, y: tip.y + 15))
+    arrow.close()
+    arrow.lineJoinStyle = .round
+    let shadow = NSShadow()
+    shadow.shadowBlurRadius = 3
+    shadow.shadowOffset = CGSize(width: 0, height: -1)
+    shadow.shadowColor = NSColor.black.withAlphaComponent(0.35)
+    NSGraphicsContext.saveGraphicsState()
+    shadow.set()
+    NSColor.systemPurple.setFill()
+    arrow.fill()
+    NSGraphicsContext.restoreGraphicsState()
+    arrow.lineWidth = 1.5
+    NSColor.white.setStroke()
+    arrow.stroke()
+  }
+}

package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift

@@ -7,6 +7,8 @@ private enum SnapshotTraversalLimits {
   static let maxDesktopApps = 24
   static let maxNodes = 1500
   static let maxDepth = 12
+  /// An app session is one app's tree; Electron alone wraps its page in about ten groups.
+  static let maxAppDepth = 48
   static let maxMenuBarBandY = 64.0
   static let maxMenuBarBandHeight = 64.0
   static let maxMenuBarExtraWidth = 256.0
@@ -45,11 +47,13 @@ struct SnapshotResponse: Encodable {
   let nodes: [SnapshotNodeResponse]
   let truncated: Bool
   let backend = "macos-helper"
+  let warnings: [String]?
 }
 
 private struct SnapshotBuildResult {
   let nodes: [SnapshotNodeResponse]
   let truncated: Bool
+  var warnings: [String]? = nil
 }
 
 private struct SnapshotContext {
@@ -64,6 +68,8 @@ private struct SnapshotTraversalState {
   var nodes: [SnapshotNodeResponse] = []
   var visited: [AXUIElement] = []
   var truncated = false
+  /// An element sat at the depth cap with children the walk left out.
+  var depthCapped = false
 }
 
 private struct MenuBarWindowFallbackCandidate {
@@ -79,6 +85,8 @@ private struct MenuBarWindowFallbackCandidate {
 func captureSnapshotResponse(surface: String, bundleId: String? = nil) throws -> SnapshotResponse {
   let result: SnapshotBuildResult
   switch surface {
+  case "app":
+    result = try snapshotSessionApp(bundleId: bundleId)
   case "frontmost-app":
     result = try snapshotFrontmostApp()
   case "desktop":
@@ -86,10 +94,101 @@ func captureSnapshotResponse(surface: String, bundleId: String? = nil) throws ->
   case "menubar":
     result = try snapshotMenuBar(bundleId: bundleId)
   default:
-    throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
+    throw HelperError.invalidArgs("snapshot requires --surface <app|frontmost-app|desktop|menubar>")
   }
 
-  return SnapshotResponse(surface: surface, nodes: result.nodes, truncated: result.truncated)
+  return SnapshotResponse(
+    surface: surface, nodes: result.nodes, truncated: result.truncated, warnings: result.warnings)
+}
+
+/// The session's own app, whether or not it is frontmost.
+private func snapshotSessionApp(bundleId: String?) throws -> SnapshotBuildResult {
+  let app = try requireSessionApplication(bundleId: bundleId)
+  let webContentReady = enableRemoteAccessibilityTree(app)
+  var state = SnapshotTraversalState()
+  _ = appendApplicationSnapshot(
+    app,
+    depth: 0,
+    parentIndex: nil,
+    surface: "app",
+    maxDepth: SnapshotTraversalLimits.maxAppDepth,
+    state: &state
+  )
+  return SnapshotBuildResult(
+    nodes: state.nodes,
+    truncated: state.truncated || state.depthCapped,
+    warnings: webContentReady
+      ? nil
+      : [
+        "The app's Chromium accessibility tree did not populate, so web content may be missing from this snapshot."
+      ]
+  )
+}
+
+/// Chromium and Electron build their accessibility tree only once a client asks for it, so an
+/// unannounced reader sees a window of empty groups. Electron answers `AXManualAccessibility`;
+/// other Chromium shells answer the VoiceOver-style `AXEnhancedUserInterface`, which native apps
+/// also honor by changing window behavior, so it is set only on a Chromium app. The tree stays on
+/// for the app's lifetime, as it does for any assistive client.
+///
+/// Whether the app's web content is readable: true for an app that is not Chromium-based or whose
+/// tree is already on. The wait is paid only by the call that turns the tree on.
+private func enableRemoteAccessibilityTree(_ app: NSRunningApplication) -> Bool {
+  guard isChromiumApplication(app) else { return true }
+  let appElement = AXUIElementCreateApplication(app.processIdentifier)
+  let attributes = ["AXManualAccessibility", "AXEnhancedUserInterface"]
+  if attributes.contains(where: { boolAttribute(appElement, attribute: $0) == true }) {
+    return true
+  }
+  guard
+    attributes.contains(where: {
+      AXUIElementSetAttributeValue(appElement, $0 as CFString, kCFBooleanTrue) == .success
+    })
+  else {
+    return false
+  }
+  return awaitPopulatedWebContent(appElement)
+}
+
+/// Chromium builds the tree after the request returns, so a walk right after enabling it would see
+/// only the wrapper groups. Bounded well inside the helper's own deadline.
+private func awaitPopulatedWebContent(_ appElement: AXUIElement) -> Bool {
+  let deadline = Date().addingTimeInterval(1.0)
+  while !hasPopulatedWebArea(appElement) {
+    guard Date() < deadline else { return false }
+    Thread.sleep(forTimeInterval: 0.05)
+  }
+  return true
+}
+
+private func hasPopulatedWebArea(_ appElement: AXUIElement) -> Bool {
+  var budget = 400
+  func visit(_ element: AXUIElement, depth: Int) -> Bool {
+    guard budget > 0, depth < 16 else { return false }
+    budget -= 1
+    let elementChildren = children(of: element)
+    if stringAttribute(element, attribute: kAXRoleAttribute as String) == "AXWebArea" {
+      return !elementChildren.isEmpty
+    }
+    return elementChildren.contains { visit($0, depth: depth + 1) }
+  }
+  return windows(of: appElement).contains { visit($0, depth: 0) }
+}
+
+/// Every Chromium embedder, Electron or a renamed framework, ships Chromium's resource pack.
+private func isChromiumApplication(_ app: NSRunningApplication) -> Bool {
+  guard let frameworksURL = app.bundleURL?.appendingPathComponent("Contents/Frameworks"),
+    let frameworks = try? FileManager.default.contentsOfDirectory(atPath: frameworksURL.path)
+  else {
+    return false
+  }
+  return frameworks.contains { framework in
+    framework.hasSuffix(".framework")
+      && FileManager.default.fileExists(
+        atPath: frameworksURL.appendingPathComponent(framework)
+          .appendingPathComponent("Resources/chrome_100_percent.pak").path
+      )
+  }
 }
 
 private func snapshotFrontmostApp() throws -> SnapshotBuildResult {
@@ -166,6 +265,7 @@ private func appendApplicationSnapshot(
   depth: Int,
   parentIndex: Int?,
   surface: String,
+  maxDepth: Int = SnapshotTraversalLimits.maxDepth,
   state: inout SnapshotTraversalState
 ) -> Bool {
   let appElement = AXUIElementCreateApplication(app.processIdentifier)
@@ -207,7 +307,8 @@ private func appendApplicationSnapshot(
         appName: app.localizedName,
         windowTitle: windowTitle
       ),
-      state: &state
+      state: &state,
+      maxDepth: maxDepth
     )
   }
 
@@ -542,6 +643,9 @@ private func appendElementSnapshot(
   )
 
   guard depth < maxDepth, !state.truncated else {
+    if depth >= maxDepth, !snapshotChildren(of: element, role: role).isEmpty {
+      state.depthCapped = true
+    }
     return index
   }
 

package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -122,6 +122,12 @@ struct AgentDeviceMacOSHelper {
       return try handleRead(arguments: Array(arguments.dropFirst()))
     case "press":
       return try handlePress(arguments: Array(arguments.dropFirst()))
+    case "type":
+      return try handleType(arguments: Array(arguments.dropFirst()))
+    case "fill":
+      return try handleFill(arguments: Array(arguments.dropFirst()))
+    case "scroll":
+      return try handleScroll(arguments: Array(arguments.dropFirst()))
     case "screenshot":
       return try handleScreenshot(arguments: Array(arguments.dropFirst()))
     case "audio-probe":
@@ -343,18 +349,16 @@ struct AgentDeviceMacOSHelper {
       .lowercased(),
       !surface.isEmpty
     else {
-      throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
+      throw HelperError.invalidArgs("snapshot requires --surface <app|frontmost-app|desktop|menubar>")
     }
 
     let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
 
     switch surface {
-    case "frontmost-app":
-      return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId))
-    case "desktop", "menubar":
+    case "app", "frontmost-app", "desktop", "menubar":
       return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId))
     default:
-      throw HelperError.invalidArgs("snapshot requires --surface <frontmost-app|desktop|menubar>")
+      throw HelperError.invalidArgs("snapshot requires --surface <app|frontmost-app|desktop|menubar>")
     }
   }
 
@@ -376,25 +380,25 @@ struct AgentDeviceMacOSHelper {
   static func handlePress(arguments: [String]) throws -> any Encodable {
     guard let rawX = optionValue(arguments: arguments, name: "--x"),
           let rawY = optionValue(arguments: arguments, name: "--y"),
-          let x = Double(rawX),
-          let y = Double(rawY)
+          let x = Double(rawX), x.isFinite,
+          let y = Double(rawY), y.isFinite
     else {
       throw HelperError.invalidArgs("press requires --x <number> --y <number>")
     }
 
-    let holdMs = try validatedPressInt(
+    let holdMs = try validatedIntOption(
       optionValue(arguments: arguments, name: "--hold-ms"),
       name: "--hold-ms",
       minimum: 0,
       default: 0
     )
-    let clicks = try validatedPressInt(
+    let clicks = try validatedIntOption(
       optionValue(arguments: arguments, name: "--clicks"),
       name: "--clicks",
       minimum: 1,
       default: 1
     )
-    let intervalMs = try validatedPressInt(
+    let intervalMs = try validatedIntOption(
       optionValue(arguments: arguments, name: "--interval-ms"),
       name: "--interval-ms",
       minimum: 0,
@@ -412,6 +416,14 @@ struct AgentDeviceMacOSHelper {
       doubleClick: doubleClick,
       intervalMs: intervalMs
     )
+    if surface == "app" {
+      let app = try requireSessionApplication(bundleId: bundleId)
+      return SuccessEnvelope(
+        data: try withGhostCursor(arguments) { cursor in
+          try pressInBackground(request, app: app, cursor: cursor)
+        }
+      )
+    }
     try pressAtPosition(request)
     return SuccessEnvelope(
       data: PressResponse(
@@ -426,6 +438,71 @@ struct AgentDeviceMacOSHelper {
     )
   }
 
+  static func handleType(arguments: [String]) throws -> any Encodable {
+    guard let text = optionValue(arguments: arguments, name: "--text") else {
+      throw HelperError.invalidArgs("type requires --text <text>")
+    }
+    let delayMs = try validatedIntOption(
+      optionValue(arguments: arguments, name: "--delay-ms"),
+      name: "--delay-ms",
+      minimum: 0,
+      default: 0
+    )
+    let app = try requireSessionApplication(
+      bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+    )
+    return SuccessEnvelope(
+      data: try withGhostCursor(arguments) { cursor in
+        try typeInBackground(text: text, delayMs: delayMs, app: app, cursor: cursor)
+      }
+    )
+  }
+
+  static func handleFill(arguments: [String]) throws -> any Encodable {
+    guard let rawX = optionValue(arguments: arguments, name: "--x"),
+          let rawY = optionValue(arguments: arguments, name: "--y"),
+          let x = Double(rawX), x.isFinite,
+          let y = Double(rawY), y.isFinite,
+          let text = optionValue(arguments: arguments, name: "--text")
+    else {
+      throw HelperError.invalidArgs("fill requires --x <number> --y <number> --text <text>")
+    }
+    let app = try requireSessionApplication(
+      bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+    )
+    return SuccessEnvelope(
+      data: try withGhostCursor(arguments) { cursor in
+        try fillInBackground(
+          point: CGPoint(x: x, y: y), text: text, app: app, cursor: cursor)
+      }
+    )
+  }
+
+  static func handleScroll(arguments: [String]) throws -> any Encodable {
+    guard let direction = optionValue(arguments: arguments, name: "--direction") else {
+      throw HelperError.invalidArgs("scroll requires --direction <up|down|left|right>")
+    }
+    let amount = try positiveDoubleOption(arguments: arguments, name: "--amount")
+    let pixels = try positiveDoubleOption(arguments: arguments, name: "--pixels")
+    if amount != nil, pixels != nil {
+      throw HelperError.invalidArgs("scroll accepts --amount or --pixels, not both")
+    }
+    let app = try requireSessionApplication(
+      bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+    )
+    return SuccessEnvelope(
+      data: try withGhostCursor(arguments) { cursor in
+        try scrollInBackground(
+          direction: direction,
+          amount: amount,
+          pixels: pixels,
+          app: app,
+          cursor: cursor
+        )
+      }
+    )
+  }
+
   static func handleScreenshot(arguments: [String]) throws -> any Encodable {
     guard let outPath = optionValue(arguments: arguments, name: "--out")?
       .trimmingCharacters(in: .whitespacesAndNewlines),
@@ -435,6 +512,13 @@ struct AgentDeviceMacOSHelper {
     }
 
     let surface = optionValue(arguments: arguments, name: "--surface")
+    if surface == "app" {
+      let app = try requireSessionApplication(
+        bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
+      )
+      try captureAppWindowScreenshot(app: app, outPath: outPath)
+      return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface))
+    }
     try captureSurfaceScreenshot(surface: surface, outPath: outPath)
     return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface))
   }
@@ -468,6 +552,21 @@ private func optionValue(arguments: [String], name: String) -> String? {
   return arguments[index + 1]
 }
 
+/// Runs a background action under the ghost cursor when the host asked for one.
+private func withGhostCursor<T>(_ arguments: [String], _ body: (GhostCursor?) throws -> T) throws -> T {
+  let cursor = arguments.contains("--ghost-cursor") ? GhostCursor.show() : nil
+  defer { cursor?.hide() }
+  return try body(cursor)
+}
+
+private func positiveDoubleOption(arguments: [String], name: String) throws -> Double? {
+  guard let raw = optionValue(arguments: arguments, name: name) else { return nil }
+  guard let value = Double(raw), value.isFinite, value > 0 else {
+    throw HelperError.invalidArgs("\(name) must be a positive number")
+  }
+  return value
+}
+
 private func intOption(arguments: [String], name: String) -> Int? {
   guard let value = optionValue(arguments: arguments, name: name) else {
     return nil
@@ -475,7 +574,7 @@ private func intOption(arguments: [String], name: String) -> Int? {
   return Int(value)
 }
 
-private func validatedPressInt(
+private func validatedIntOption(
   _ raw: String?,
   name: String,
   minimum: Int,
@@ -485,22 +584,30 @@ private func validatedPressInt(
     return fallback
   }
   guard let value = Int(raw), value >= minimum else {
-    throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)")
+    throw HelperError.invalidArgs("\(name) must be an integer of at least \(minimum)")
   }
   return value
 }
 
 private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String {
   let targetApp: NSRunningApplication?
-  if surface == "frontmost-app" || (surface == nil && bundleId != nil) {
+  if surface == "app" {
+    targetApp = try requireSessionApplication(bundleId: bundleId)
+  } else if surface == "frontmost-app" || (surface == nil && bundleId != nil) {
     targetApp = try resolveTargetApplication(bundleId: bundleId, surface: surface)
   } else {
     targetApp = nil
   }
 
-  let systemWide = AXUIElementCreateSystemWide()
+  // An app session's window may sit behind other apps, so it is hit-tested on its own.
+  let hitRoot =
+    if surface == "app", let targetApp {
+      AXUIElementCreateApplication(targetApp.processIdentifier)
+    } else {
+      AXUIElementCreateSystemWide()
+    }
   var hitElement: AXUIElement?
-  guard AXUIElementCopyElementAtPosition(systemWide, Float(x), Float(y), &hitElement) == .success,
+  guard AXUIElementCopyElementAtPosition(hitRoot, Float(x), Float(y), &hitElement) == .success,
         let hitElement
   else {
     throw HelperError.commandFailed("read did not resolve an accessibility element")
@@ -567,18 +674,25 @@ private func captureSurfaceScreenshot(surface: String?, outPath: String) throws
   semaphore.wait()
 
   if let error = capturedError as NSError? {
-    if error.domain == "com.apple.ScreenCaptureKit.SCStreamErrorDomain", error.code == -3801 {
-      throw HelperError.commandFailed(
-        "screenshot requires Screen Recording permission on macOS desktop and menubar surfaces",
-        details: ["surface": surface ?? "", "permission": "screen-recording"]
-      )
-    }
-    throw HelperError.commandFailed("screenshot failed", details: ["error": error.localizedDescription])
+    throw screenshotFailure(error, surface: surface)
   }
   guard let capturedImage else {
     throw HelperError.commandFailed("screenshot failed")
   }
+  try writePNG(capturedImage, to: outPath)
+}
+
+func screenshotFailure(_ error: NSError, surface: String?) -> HelperError {
+  if error.domain == "com.apple.ScreenCaptureKit.SCStreamErrorDomain", error.code == -3801 {
+    return HelperError.commandFailed(
+      "screenshot requires Screen Recording permission on the macOS \(surface ?? "desktop") surface",
+      details: ["surface": surface ?? "", "permission": "screen-recording"]
+    )
+  }
+  return HelperError.commandFailed("screenshot failed", details: ["error": error.localizedDescription])
+}
 
+func writePNG(_ capturedImage: CGImage, to outPath: String) throws {
   let outputURL = URL(fileURLWithPath: outPath)
   if let parent = outputURL.deletingLastPathComponent().path.removingPercentEncoding, !parent.isEmpty {
     try FileManager.default.createDirectory(atPath: parent, withIntermediateDirectories: true)

package/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift

@@ -0,0 +1,88 @@
+import XCTest
+
+@testable import AgentDeviceMacOSHelper
+
+private struct ScrollGestureFixture: Decodable {
+  struct Constants: Decodable {
+    let defaultScrollAmount: Double
+  }
+  struct Expected: Decodable {
+    let pixels: Double
+  }
+  struct Case: Decodable {
+    let name: String
+    let direction: String
+    let amount: Double?
+    let pixels: Double?
+    let referenceWidth: Double
+    let referenceHeight: Double
+    let expected: Expected
+  }
+
+  let constants: Constants
+  let cases: [Case]
+}
+
+private struct HelperOutcomesFixture: Decodable {
+  let refusalReasons: [String]
+  let deliveryMechanisms: [String]
+}
+
+/// `#filePath` is `<repo>/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/<file>`.
+private let repoRoot = URL(fileURLWithPath: #filePath)
+  .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+  .deletingLastPathComponent().deletingLastPathComponent()
+
+final class BackgroundInteractionTests: XCTestCase {
+  /// The host keys refusals and mechanisms on these strings; the table is their one declaration.
+  func testOutcomeVocabularyMatchesTheSharedTable() throws {
+    let fixture = try JSONDecoder().decode(
+      HelperOutcomesFixture.self,
+      from: Data(
+        contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json"))
+    )
+    XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons)
+    XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms)
+  }
+
+  /// The native backend scrolls as far as the runner does: its travel agrees with every case of
+  /// the cross-language table the runner and the TypeScript planner are held to.
+  func testScrollTravelMatchesTheSharedScrollGestureTable() throws {
+    let fixtureURL = repoRoot.appendingPathComponent("contracts/fixtures/scroll-gesture.json")
+    let fixture = try JSONDecoder().decode(
+      ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL))
+    XCTAssertFalse(fixture.cases.isEmpty)
+    for testCase in fixture.cases {
+      let isVertical = testCase.direction == "up" || testCase.direction == "down"
+      let travel = scrollTravelPixels(
+        axisLength: isVertical ? testCase.referenceHeight : testCase.referenceWidth,
+        amount: testCase.amount,
+        pixels: testCase.pixels
+      )
+      XCTAssertEqual(travel, testCase.expected.pixels, testCase.name)
+    }
+    XCTAssertEqual(
+      scrollTravelPixels(axisLength: 1000, amount: nil, pixels: nil),
+      (1000 * fixture.constants.defaultScrollAmount).rounded(),
+      "an unspecified distance travels the table's default amount"
+    )
+  }
+
+  func testScrollBarValueMovesByTheTravelShareOfTheOverflow() {
+    XCTAssertEqual(scrollBarValue(current: 0, signedTravel: 300, overflow: 1200), 0.25)
+    XCTAssertEqual(scrollBarValue(current: 0.5, signedTravel: -300, overflow: 1200), 0.25)
+  }
+
+  func testScrollBarValueStopsAtTheEnds() {
+    XCTAssertEqual(scrollBarValue(current: 0.9, signedTravel: 600, overflow: 1200), 1)
+    XCTAssertEqual(scrollBarValue(current: 0.1, signedTravel: -600, overflow: 1200), 0)
+  }
+
+  /// Chromium marks wrapper groups pressable; only a control role may win the press outright.
+  func testWrapperGroupsAreNotPressableControls() {
+    XCTAssertTrue(isPressableControlRole("AXButton"))
+    XCTAssertTrue(isPressableControlRole("AXLink"))
+    XCTAssertFalse(isPressableControlRole("AXGroup"))
+    XCTAssertFalse(isPressableControlRole("AXWebArea"))
+  }
+}