agent-device 0.18.3 → 0.19.1

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -0,0 +1,155 @@
+import Foundation
+import XCTest
+
+enum RunnerCommandLifecycleState: String {
+  case notAccepted
+  case accepted
+  case started
+  case completed
+  case failed
+}
+
+struct RunnerCommandJournalEntry {
+  let commandId: String
+  let command: String
+  var state: RunnerCommandLifecycleState
+  var responseOk: Bool?
+  var responseJson: String?
+  var error: ErrorPayload?
+}
+
+final class RunnerCommandJournal {
+  private let lock = NSLock()
+  private let maxEntries = 64
+  private let maxResponseJsonBytes = 16 * 1024
+  private var entries: [String: RunnerCommandJournalEntry] = [:]
+  private var order: [String] = []
+
+  func accept(command: Command) {
+    guard let commandId = normalizedCommandId(command.commandId) else { return }
+    lock.lock()
+    defer { lock.unlock() }
+    entries[commandId] = RunnerCommandJournalEntry(
+      commandId: commandId,
+      command: command.command.rawValue,
+      state: .accepted,
+      responseOk: nil,
+      responseJson: nil,
+      error: nil
+    )
+    order.removeAll { $0 == commandId }
+    order.append(commandId)
+    pruneIfNeeded()
+  }
+
+  func start(command: Command) {
+    update(command: command, state: .started, responseOk: nil, responseJson: nil, error: nil)
+  }
+
+  func finish(command: Command, response: Response) {
+    update(
+      command: command,
+      state: response.ok ? .completed : .failed,
+      responseOk: response.ok,
+      responseJson: encodeResponseJson(command: command, response: response),
+      error: response.error
+    )
+  }
+
+  func fail(command: Command, error: Error) {
+    update(
+      command: command,
+      state: .failed,
+      responseOk: nil,
+      responseJson: nil,
+      error: ErrorPayload(message: error.localizedDescription)
+    )
+  }
+
+  func status(commandId: String) -> DataPayload {
+    guard let normalized = normalizedCommandId(commandId) else {
+      return DataPayload(lifecycleState: RunnerCommandLifecycleState.notAccepted.rawValue)
+    }
+    lock.lock()
+    let entry = entries[normalized]
+    lock.unlock()
+    guard let entry else {
+      return DataPayload(
+        commandId: normalized,
+        lifecycleState: RunnerCommandLifecycleState.notAccepted.rawValue
+      )
+    }
+    return DataPayload(
+      commandId: entry.commandId,
+      lifecycleState: entry.state.rawValue,
+      lifecycleCommand: entry.command,
+      lifecycleResponseOk: entry.responseOk,
+      lifecycleResponseJson: entry.responseJson,
+      lifecycleErrorCode: entry.error?.code,
+      lifecycleErrorMessage: entry.error?.message,
+      lifecycleErrorHint: entry.error?.hint
+    )
+  }
+
+  private func update(
+    command: Command,
+    state: RunnerCommandLifecycleState,
+    responseOk: Bool?,
+    responseJson: String?,
+    error: ErrorPayload?
+  ) {
+    guard let commandId = normalizedCommandId(command.commandId) else { return }
+    lock.lock()
+    defer { lock.unlock() }
+    var entry = entries[commandId] ?? RunnerCommandJournalEntry(
+      commandId: commandId,
+      command: command.command.rawValue,
+      state: .accepted,
+      responseOk: nil,
+      responseJson: nil,
+      error: nil
+    )
+    entry.state = state
+    entry.responseOk = responseOk
+    entry.responseJson = responseJson
+    entry.error = error
+    entries[commandId] = entry
+    order.removeAll { $0 == commandId }
+    order.append(commandId)
+    pruneIfNeeded()
+  }
+
+  private func pruneIfNeeded() {
+    while order.count > maxEntries {
+      let removed = order.removeFirst()
+      entries.removeValue(forKey: removed)
+    }
+  }
+
+  private func normalizedCommandId(_ value: String?) -> String? {
+    guard let value else { return nil }
+    let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
+    return trimmed.isEmpty ? nil : trimmed
+  }
+
+  private func encodeResponseJson(command: Command, response: Response) -> String? {
+    guard shouldRetainResponseJson(command: command) else { return nil }
+    guard let data = try? JSONEncoder().encode(response) else { return nil }
+    guard data.count <= maxResponseJsonBytes else { return nil }
+    return String(data: data, encoding: .utf8)
+  }
+
+  private func shouldRetainResponseJson(command: Command) -> Bool {
+    switch command.command {
+    case .snapshot, .screenshot:
+      return false
+    case .tap, .mouseClick, .longPress, .drag,
+         .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .back,
+         .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn,
+         .alert, .pinch, .sequence, .rotateGesture, .transformGesture, .recordStart, .recordStop,
+         .status, .uptime, .shutdown:
+      return true
+    }
+  }
+}
+

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift

@@ -0,0 +1,67 @@
+import XCTest
+
+struct FlatSnapshotFilterNode {
+  let isRoot: Bool
+  let label: String
+  let identifier: String
+  let valueText: String?
+  let visible: Bool
+
+  func matchesScope(_ scope: String) -> Bool {
+    let haystack = [label, identifier, valueText ?? ""].joined(separator: "\n")
+    return haystack.localizedCaseInsensitiveContains(scope)
+  }
+}
+
+struct FlatSnapshotFilterDecision {
+  let include: Bool
+  let insideMatchedScope: Bool
+}
+
+extension RunnerTests {
+  func flatSnapshotFilterDecision(
+    _ node: FlatSnapshotFilterNode,
+    options: SnapshotOptions,
+    insideMatchedScope: Bool
+  ) -> FlatSnapshotFilterDecision {
+    let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines)
+    let scopeActive = scope?.isEmpty == false
+    let matchesScope: Bool
+    if scopeActive, let scope {
+      matchesScope = node.matchesScope(scope)
+    } else {
+      matchesScope = false
+    }
+    let nowInsideScope = insideMatchedScope || matchesScope
+
+    let include: Bool
+    if node.isRoot {
+      include = true
+    } else if scopeActive && !nowInsideScope {
+      include = false
+    } else if options.interactiveOnly && !node.visible {
+      include = false
+    } else {
+      include = true
+    }
+
+    return FlatSnapshotFilterDecision(include: include, insideMatchedScope: nowInsideScope)
+  }
+
+  func privateAXInteractiveCandidate(rawElementType: Int) -> Bool {
+    guard let type = flatSnapshotElementType(rawElementType: rawElementType) else {
+      return false
+    }
+    return interactiveTypes.contains(type) || Self.scrollContainerTypes.contains(type)
+  }
+
+  func flatSnapshotElementType(rawElementType: Int) -> XCUIElement.ElementType? {
+    guard let raw = UInt(exactly: rawElementType),
+      let type = XCUIElement.ElementType(rawValue: raw)
+    else {
+      return nil
+    }
+    return type
+  }
+
+}

package/{apple-runner → dist/apple-runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -36,34 +36,21 @@ extension RunnerTests {
     let y2: Double
   }
 
+  struct SynthesizedDragPlan {
+    let points: DragPoints
+    let context: SynthesizedCoordinateContext
+
+    var referenceFrame: CGRect {
+      context.referenceFrame
+    }
+  }
+
   struct SelectorElementMatch {
     let element: XCUIElement?
     let isAmbiguous: Bool
     let usedNonHittableFallback: Bool
   }
 
-  // MARK: - Navigation Gestures
-
-  func tapInAppBackControl(app: XCUIApplication) -> Bool {
-#if os(macOS)
-    if let back = macOSNavigationBackElement(app: app) {
-      tapElementCenter(app: app, element: back)
-      return true
-    }
-    return false
-#elseif os(tvOS)
-    _ = pressTvRemote(.menu)
-    return true
-#else
-    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
-    if let back = buttons.first(where: { $0.isHittable }) {
-      back.tap()
-      return true
-    }
-    return false
-#endif
-  }
-
   func performBackGesture(app: XCUIApplication) {
     if pressTvRemote(.menu) {
       return
@@ -200,16 +187,32 @@ extension RunnerTests {
     )
   }
 
+  // Maestro-compat gate for the non-hittable coordinate fallback: an element
+  // with no frame at all cannot be coordinate-tapped, otherwise the decision
+  // is the shared TapPointPolicy center-in-frame rule (golden parity table
+  // with the TS twin). app.frame is the frame source here — replay taps
+  // resolved bounds Maestro-style, so the union frame is intentional.
   private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool {
     let frame = element.frame
     if frame.isEmpty {
       return false
     }
-    let appFrame = app.frame
-    if appFrame.isEmpty {
-      return true
+    return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: app.frame)
+  }
+
+  // The tappable on-screen viewport. app.frame is unsuitable: it unions
+  // transformed subtrees, so a closed drawer at negative x inflates it and
+  // out-of-window coordinates still pass containment. Falls back to app.frame
+  // when no window frame is readable.
+  func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
+    let window = app.windows.element(boundBy: 0)
+    if window.exists {
+      let frame = window.frame
+      if !frame.isEmpty {
+        return frame
+      }
     }
-    return appFrame.contains(CGPoint(x: frame.midX, y: frame.midY))
+    return app.frame
   }
 
   func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
@@ -771,11 +774,24 @@ extension RunnerTests {
     y: Double,
     x2: Double,
     y2: Double,
-    durationMs: Double
+    durationMs: Double,
+    context: SynthesizedCoordinateContext? = nil
   ) -> RunnerInteractionOutcome {
 #if os(iOS)
+    guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
+      return .unsupported(
+        message: "synthesized coordinate drag requires finite coordinates",
+        hint: "Retry with finite x, y, x2, and y2 values."
+      )
+    }
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    let frame = app.frame
+    guard let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.synthesizedDrag)) else {
+      return .unsupported(
+        message: "synthesized coordinate drag could not resolve a finite screen frame",
+        hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+      )
+    }
+    let frame = context.referenceFrame
     let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
     let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
     if let message = RunnerSynthesizedGesture.synthesizeSwipe(
@@ -788,7 +804,7 @@ extension RunnerTests {
     ) {
       return .unsupported(
         message: message,
-        hint: "Falling back to XCTest coordinate drag may be slower; update Xcode if this persists."
+        hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists."
       )
     }
     return .performed
@@ -805,10 +821,28 @@ extension RunnerTests {
 #endif
   }
 
-  func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+  func synthesizedTapAt(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    context: SynthesizedCoordinateContext? = nil
+  ) -> RunnerInteractionOutcome {
 #if os(iOS)
+    guard x.isFinite, y.isFinite else {
+      return .unsupported(
+        message: "synthesized coordinate tap requires finite coordinates",
+        hint: "Retry with finite x and y values."
+      )
+    }
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation)
+    guard let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.coordinateTap)) else {
+      return .unsupported(
+        message: "synthesized coordinate tap could not resolve a finite screen frame",
+        hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+      )
+    }
+    let frame = context.referenceFrame
+    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
     if let message = RunnerSynthesizedGesture.synthesizeTap(
       withApplication: app,
       x: Double(point.x),
@@ -978,6 +1012,114 @@ extension RunnerTests {
 #endif
   }
 
+  func axFreeSynthesizedDragPlan(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    context: SynthesizedCoordinateContext? = nil
+  ) -> SynthesizedDragPlan? {
+#if os(iOS)
+    let context = context ?? synthesizedCoordinateContext(policy: synthesizedGesturePolicy(.synthesizedDrag))
+    guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
+      let context
+    else {
+      return nil
+    }
+    let points = keyboardAvoidingSynthesizedDragPoints(
+      app: app,
+      x: x,
+      y: y,
+      x2: x2,
+      y2: y2,
+      context: context
+    )
+    return SynthesizedDragPlan(
+      points: points,
+      context: context
+    )
+#else
+    return nil
+#endif
+  }
+
+  func axFreeDragVisualizationFrame(
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    referenceFrame: CGRect
+  ) -> DragVisualizationFrame {
+    return DragVisualizationFrame(
+      x: x,
+      y: y,
+      x2: x2,
+      y2: y2,
+      referenceWidth: Double(referenceFrame.width),
+      referenceHeight: Double(referenceFrame.height)
+    )
+  }
+
+  func synthesizedCoordinateContext(policy: SynthesizedGesturePolicy) -> SynthesizedCoordinateContext? {
+#if os(iOS)
+    let health = runnerAccessibilityHealth
+    guard let referenceFrame = synthesizedScreenshotReferenceFrame(
+      screenshotSize: { XCUIScreen.main.screenshot().image.size }
+    ) else {
+      return nil
+    }
+    return SynthesizedCoordinateContext(
+      referenceFrame: referenceFrame,
+      keyboardPolicy: policy.keyboardPolicy,
+      fallbackPolicy: policy.fallbackPolicy,
+      accessibilityHealth: health
+    )
+#else
+    return nil
+#endif
+  }
+
+  func synthesizedScreenshotReferenceFrame(screenshotSize: () -> CGSize) -> CGRect? {
+    let screenshotSize = screenshotSize()
+    guard screenshotSize.width.isFinite, screenshotSize.height.isFinite,
+      screenshotSize.width > 0,
+      screenshotSize.height > 0
+    else {
+      return nil
+    }
+    return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height)
+  }
+
+  func synthesizedFrameAvoidingKeyboardWhenAllowed(
+    app: XCUIApplication,
+    context: SynthesizedCoordinateContext
+  ) -> CGRect {
+#if os(iOS)
+    guard context.allowsKeyboardProbe else { return context.referenceFrame }
+    return frameAvoidingKeyboard(app: app, frame: context.referenceFrame)
+#else
+    return context.referenceFrame
+#endif
+  }
+
+  func keyboardAvoidingSynthesizedDragPoints(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    context: SynthesizedCoordinateContext
+  ) -> DragPoints {
+#if os(iOS)
+    guard context.allowsKeyboardProbe else {
+      return DragPoints(x: x, y: y, x2: x2, y2: y2)
+    }
+    return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
+#else
+    return DragPoints(x: x, y: y, x2: x2, y2: y2)
+#endif
+  }
 
   func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {
@@ -1211,95 +1353,4 @@ extension RunnerTests {
   }
 #endif
 
-  private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
-    let frame = element.frame
-    if !frame.isEmpty {
-      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
-      return
-    }
-#if !os(tvOS)
-    element.tap()
-#endif
-  }
-
-  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
-    let predicate = NSPredicate(
-      format: "identifier == %@ OR label == %@",
-      "go back",
-      "Back"
-    )
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
-  }
-
-#if AGENT_DEVICE_RUNNER_UNIT_TESTS
-  // Identity in portrait/unknown, 90° per landscape, 180° upside-down.
-  func testNativeSynthesizedPointRotatesByInterfaceOrientation() {
-    let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210)
-    let landscape = CGRect(x: 0, y: 0, width: 1210, height: 834)
-    let offsetLandscape = CGRect(x: 10, y: 20, width: 1210, height: 834)
-    // (frame, UIInterfaceOrientation, expected native point) for a tap at (170, 268).
-    let cases: [(CGRect, Int, CGPoint)] = [
-      (portrait, RunnerInterfaceOrientation.portrait, CGPoint(x: 170, y: 268)),
-      (landscape, RunnerInterfaceOrientation.landscapeRight, CGPoint(x: 566, y: 170)),
-      (landscape, RunnerInterfaceOrientation.landscapeLeft, CGPoint(x: 268, y: 1040)),
-      (portrait, RunnerInterfaceOrientation.portraitUpsideDown, CGPoint(x: 664, y: 942)),
-      (portrait, RunnerInterfaceOrientation.unknown, CGPoint(x: 170, y: 268)),
-    ]
-    for (frame, orientation, expected) in cases {
-      XCTAssertEqual(
-        nativeSynthesizedPoint(orientedX: 170, orientedY: 268, in: frame, interfaceOrientation: orientation),
-        expected,
-        "interfaceOrientation \(orientation)"
-      )
-    }
-    XCTAssertEqual(
-      nativeSynthesizedPoint(
-        orientedX: 180,
-        orientedY: 288,
-        in: offsetLandscape,
-        interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft
-      ),
-      CGPoint(x: 268, y: 1040),
-      "non-zero frame origin is localized before rotation"
-    )
-  }
-
-  func testNativeSynthesizedVectorRotatesByInterfaceOrientation() {
-    let cases: [(Int, CGVector)] = [
-      (RunnerInterfaceOrientation.portrait, CGVector(dx: 40, dy: -20)),
-      (RunnerInterfaceOrientation.landscapeRight, CGVector(dx: 20, dy: 40)),
-      (RunnerInterfaceOrientation.landscapeLeft, CGVector(dx: -20, dy: -40)),
-      (RunnerInterfaceOrientation.portraitUpsideDown, CGVector(dx: -40, dy: 20)),
-      (RunnerInterfaceOrientation.unknown, CGVector(dx: 40, dy: -20)),
-    ]
-    for (orientation, expected) in cases {
-      let vector = nativeSynthesizedVector(orientedDx: 40, orientedDy: -20, interfaceOrientation: orientation)
-      XCTAssertEqual(vector.dx, expected.dx, "dx interfaceOrientation \(orientation)")
-      XCTAssertEqual(vector.dy, expected.dy, "dy interfaceOrientation \(orientation)")
-    }
-  }
-
-  func testDesktopScrollWheelDeltasMapDirections() throws {
-    XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "up", pixels: 120)).vertical, 120)
-    XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "down", pixels: 120)).vertical, -120)
-    XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "left", pixels: 120)).horizontal, 120)
-    XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "right", pixels: 120)).horizontal, -120)
-    XCTAssertNil(desktopScrollWheelDeltas(direction: "diagonal", pixels: 120))
-  }
-
-  func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() throws {
-    let events = try XCTUnwrap(desktopScrollWheelDeltaEvents(direction: "down", pixels: 200, durationMs: 50))
-    XCTAssertEqual(events.count, 4)
-    XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200)
-    XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0)
-    XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0)
-  }
-
-  func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() throws {
-    let events = try XCTUnwrap(desktopScrollWheelDeltaEvents(direction: "down", pixels: 200, durationMs: 0))
-    XCTAssertEqual(events.count, 1)
-    XCTAssertEqual(events.first?.vertical, -200)
-  }
-#endif
 }