agent-device 0.7.4 → 0.7.5

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -0,0 +1,359 @@
+import XCTest
+
+extension RunnerTests {
+  // MARK: - Snapshot Entry
+
+  func elementTypeName(_ type: XCUIElement.ElementType) -> String {
+    switch type {
+    case .application: return "Application"
+    case .window: return "Window"
+    case .button: return "Button"
+    case .cell: return "Cell"
+    case .staticText: return "StaticText"
+    case .textField: return "TextField"
+    case .textView: return "TextView"
+    case .secureTextField: return "SecureTextField"
+    case .switch: return "Switch"
+    case .slider: return "Slider"
+    case .link: return "Link"
+    case .image: return "Image"
+    case .navigationBar: return "NavigationBar"
+    case .tabBar: return "TabBar"
+    case .collectionView: return "CollectionView"
+    case .table: return "Table"
+    case .scrollView: return "ScrollView"
+    case .searchField: return "SearchField"
+    case .segmentedControl: return "SegmentedControl"
+    case .stepper: return "Stepper"
+    case .picker: return "Picker"
+    case .checkBox: return "CheckBox"
+    case .menuItem: return "MenuItem"
+    case .other: return "Other"
+    default:
+      switch type.rawValue {
+      case 19:
+        return "Keyboard"
+      case 20:
+        return "Key"
+      case 24:
+        return "SearchField"
+      default:
+        return "Element(\(type.rawValue))"
+      }
+    }
+  }
+
+  func snapshotFast(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
+    if let blocking = blockingSystemAlertSnapshot() {
+      return blocking
+    }
+
+    var nodes: [SnapshotNode] = []
+    var truncated = false
+    let maxDepth = options.depth ?? Int.max
+    let viewport = app.frame
+    let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
+
+    let rootSnapshot: XCUIElementSnapshot
+    do {
+      rootSnapshot = try queryRoot.snapshot()
+    } catch {
+      return DataPayload(nodes: nodes, truncated: truncated)
+    }
+
+    let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
+    let rootLaterNodes = laterSnapshots(
+      for: rootSnapshot,
+      in: flatSnapshots,
+      ranges: snapshotRanges
+    )
+    let rootLabel = aggregatedLabel(for: rootSnapshot) ?? rootSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let rootIdentifier = rootSnapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+    let rootValue = snapshotValueText(rootSnapshot)
+    let rootHittable = computedSnapshotHittable(rootSnapshot, viewport: viewport, laterNodes: rootLaterNodes)
+    nodes.append(
+      SnapshotNode(
+        index: 0,
+        type: elementTypeName(rootSnapshot.elementType),
+        label: rootLabel.isEmpty ? nil : rootLabel,
+        identifier: rootIdentifier.isEmpty ? nil : rootIdentifier,
+        value: rootValue,
+        rect: SnapshotRect(
+          x: Double(rootSnapshot.frame.origin.x),
+          y: Double(rootSnapshot.frame.origin.y),
+          width: Double(rootSnapshot.frame.size.width),
+          height: Double(rootSnapshot.frame.size.height),
+        ),
+        enabled: rootSnapshot.isEnabled,
+        hittable: rootHittable,
+        depth: 0,
+      )
+    )
+
+    var seen = Set<String>()
+    var stack: [(XCUIElementSnapshot, Int, Int)] = rootSnapshot.children.map { ($0, 1, 1) }
+
+    while let (snapshot, depth, visibleDepth) = stack.popLast() {
+      if nodes.count >= fastSnapshotLimit {
+        truncated = true
+        break
+      }
+      if let limit = options.depth, depth > limit { continue }
+
+      let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+      let valueText = snapshotValueText(snapshot)
+      let laterNodes = laterSnapshots(
+        for: snapshot,
+        in: flatSnapshots,
+        ranges: snapshotRanges
+      )
+      let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
+      let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
+      if !isVisibleInViewport(snapshot.frame, viewport) && !hasContent {
+        continue
+      }
+
+      let include = shouldInclude(
+        snapshot: snapshot,
+        label: label,
+        identifier: identifier,
+        valueText: valueText,
+        options: options,
+        hittable: hittable
+      )
+
+      let key = "\(snapshot.elementType)-\(label)-\(identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
+      let isDuplicate = seen.contains(key)
+      if !isDuplicate {
+        seen.insert(key)
+      }
+
+      if depth < maxDepth {
+        let nextVisibleDepth = include && !isDuplicate ? visibleDepth + 1 : visibleDepth
+        for child in snapshot.children.reversed() {
+          stack.append((child, depth + 1, nextVisibleDepth))
+        }
+      }
+
+      if !include || isDuplicate { continue }
+
+      nodes.append(
+        SnapshotNode(
+          index: nodes.count,
+          type: elementTypeName(snapshot.elementType),
+          label: label.isEmpty ? nil : label,
+          identifier: identifier.isEmpty ? nil : identifier,
+          value: valueText,
+          rect: SnapshotRect(
+            x: Double(snapshot.frame.origin.x),
+            y: Double(snapshot.frame.origin.y),
+            width: Double(snapshot.frame.size.width),
+            height: Double(snapshot.frame.size.height),
+          ),
+          enabled: snapshot.isEnabled,
+          hittable: hittable,
+          depth: min(maxDepth, visibleDepth),
+        )
+      )
+
+    }
+
+    return DataPayload(nodes: nodes, truncated: truncated)
+  }
+
+  func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
+    if let blocking = blockingSystemAlertSnapshot() {
+      return blocking
+    }
+
+    let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
+    var nodes: [SnapshotNode] = []
+    var truncated = false
+    let viewport = app.frame
+
+    let rootSnapshot: XCUIElementSnapshot
+    do {
+      rootSnapshot = try queryRoot.snapshot()
+    } catch {
+      return DataPayload(nodes: nodes, truncated: truncated)
+    }
+
+    let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot)
+
+    func walk(_ snapshot: XCUIElementSnapshot, depth: Int) {
+      if nodes.count >= maxSnapshotElements {
+        truncated = true
+        return
+      }
+      if let limit = options.depth, depth > limit { return }
+      if !isVisibleInViewport(snapshot.frame, viewport) { return }
+
+      let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+      let valueText = snapshotValueText(snapshot)
+      let laterNodes = laterSnapshots(
+        for: snapshot,
+        in: flatSnapshots,
+        ranges: snapshotRanges
+      )
+      let hittable = computedSnapshotHittable(snapshot, viewport: viewport, laterNodes: laterNodes)
+      if shouldInclude(
+        snapshot: snapshot,
+        label: label,
+        identifier: identifier,
+        valueText: valueText,
+        options: options,
+        hittable: hittable
+      ) {
+        nodes.append(
+          SnapshotNode(
+            index: nodes.count,
+            type: elementTypeName(snapshot.elementType),
+            label: label.isEmpty ? nil : label,
+            identifier: identifier.isEmpty ? nil : identifier,
+            value: valueText,
+            rect: snapshotRect(from: snapshot.frame),
+            enabled: snapshot.isEnabled,
+            hittable: hittable,
+            depth: depth,
+          )
+        )
+      }
+
+      let children = snapshot.children
+      for child in children {
+        walk(child, depth: depth + 1)
+        if truncated { return }
+      }
+    }
+
+    walk(rootSnapshot, depth: 0)
+    return DataPayload(nodes: nodes, truncated: truncated)
+  }
+
+  func snapshotRect(from frame: CGRect) -> SnapshotRect {
+    return SnapshotRect(
+      x: Double(frame.origin.x),
+      y: Double(frame.origin.y),
+      width: Double(frame.size.width),
+      height: Double(frame.size.height)
+    )
+  }
+
+  // MARK: - Snapshot Filtering
+
+  private func shouldInclude(
+    snapshot: XCUIElementSnapshot,
+    label: String,
+    identifier: String,
+    valueText: String?,
+    options: SnapshotOptions,
+    hittable: Bool
+  ) -> Bool {
+    let type = snapshot.elementType
+    let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
+    if options.compact && type == .other && !hasContent && !hittable {
+      if snapshot.children.count <= 1 { return false }
+    }
+    if options.interactiveOnly {
+      if interactiveTypes.contains(type) { return true }
+      if hittable && type != .other { return true }
+      if hasContent { return true }
+      return false
+    }
+    if options.compact {
+      return hasContent || hittable
+    }
+    return true
+  }
+
+  private func computedSnapshotHittable(
+    _ snapshot: XCUIElementSnapshot,
+    viewport: CGRect,
+    laterNodes: ArraySlice<XCUIElementSnapshot>
+  ) -> Bool {
+    guard snapshot.isEnabled else { return false }
+    let frame = snapshot.frame
+    if frame.isNull || frame.isEmpty { return false }
+    let center = CGPoint(x: frame.midX, y: frame.midY)
+    if !viewport.contains(center) { return false }
+    for node in laterNodes {
+      if !isOccludingType(node.elementType) { continue }
+      let nodeFrame = node.frame
+      if nodeFrame.isNull || nodeFrame.isEmpty { continue }
+      if nodeFrame.contains(center) { return false }
+    }
+    return true
+  }
+
+  private func isOccludingType(_ type: XCUIElement.ElementType) -> Bool {
+    switch type {
+    case .application, .window:
+      return false
+    default:
+      return true
+    }
+  }
+
+  private func flattenedSnapshots(
+    _ root: XCUIElementSnapshot
+  ) -> ([XCUIElementSnapshot], [ObjectIdentifier: (Int, Int)]) {
+    var ordered: [XCUIElementSnapshot] = []
+    var ranges: [ObjectIdentifier: (Int, Int)] = [:]
+
+    @discardableResult
+    func visit(_ snapshot: XCUIElementSnapshot) -> Int {
+      let start = ordered.count
+      ordered.append(snapshot)
+      var end = start
+      for child in snapshot.children {
+        end = max(end, visit(child))
+      }
+      ranges[ObjectIdentifier(snapshot)] = (start, end)
+      return end
+    }
+
+    _ = visit(root)
+    return (ordered, ranges)
+  }
+
+  private func laterSnapshots(
+    for snapshot: XCUIElementSnapshot,
+    in ordered: [XCUIElementSnapshot],
+    ranges: [ObjectIdentifier: (Int, Int)]
+  ) -> ArraySlice<XCUIElementSnapshot> {
+    guard let (_, subtreeEnd) = ranges[ObjectIdentifier(snapshot)] else {
+      return ordered.suffix(from: ordered.count)
+    }
+    let nextIndex = subtreeEnd + 1
+    if nextIndex >= ordered.count {
+      return ordered.suffix(from: ordered.count)
+    }
+    return ordered.suffix(from: nextIndex)
+  }
+
+  private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? {
+    guard let value = snapshot.value else { return nil }
+    let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
+    return text.isEmpty ? nil : text
+  }
+
+  private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
+    if depth > 4 { return nil }
+    let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    if !text.isEmpty { return text }
+    if let valueText = snapshotValueText(snapshot) { return valueText }
+    for child in snapshot.children {
+      if let childLabel = aggregatedLabel(for: child, depth: depth + 1) {
+        return childLabel
+      }
+    }
+    return nil
+  }
+
+  private func isVisibleInViewport(_ rect: CGRect, _ viewport: CGRect) -> Bool {
+    if rect.isNull || rect.isEmpty { return false }
+    return rect.intersects(viewport)
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

@@ -0,0 +1,220 @@
+import XCTest
+
+extension RunnerTests {
+  // MARK: - Blocking System Modal Snapshot
+
+  func blockingSystemAlertSnapshot() -> DataPayload? {
+    guard let modal = firstBlockingSystemModal(in: springboard) else {
+      return nil
+    }
+    let actions = actionableElements(in: modal)
+    guard !actions.isEmpty else {
+      return nil
+    }
+
+    let title = preferredSystemModalTitle(modal)
+    guard let modalNode = safeMakeSnapshotNode(
+      element: modal,
+      index: 0,
+      type: "Alert",
+      labelOverride: title,
+      identifierOverride: modal.identifier,
+      depth: 0,
+      hittableOverride: true
+    ) else {
+      return nil
+    }
+    var nodes: [SnapshotNode] = [modalNode]
+
+    for action in actions {
+      guard let actionNode = safeMakeSnapshotNode(
+        element: action,
+        index: nodes.count,
+        type: elementTypeName(action.elementType),
+        depth: 1,
+        hittableOverride: true
+      ) else {
+        continue
+      }
+      nodes.append(actionNode)
+    }
+
+    return DataPayload(nodes: nodes, truncated: false)
+  }
+
+  private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
+    let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
+    let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
+      if disableSafeProbe {
+        return fetch()
+      }
+      return self.safeElementsQuery(fetch)
+    }
+
+    let alerts = queryElements {
+      springboard.alerts.allElementsBoundByIndex
+    }
+    for alert in alerts {
+      if safeIsBlockingSystemModal(alert, in: springboard) {
+        return alert
+      }
+    }
+
+    let sheets = queryElements {
+      springboard.sheets.allElementsBoundByIndex
+    }
+    for sheet in sheets {
+      if safeIsBlockingSystemModal(sheet, in: springboard) {
+        return sheet
+      }
+    }
+
+    return nil
+  }
+
+  private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
+    var elements: [XCUIElement] = []
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      elements = fetch()
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_MODAL_QUERY_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return []
+    }
+    return elements
+  }
+
+  private func safeIsBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
+    var isBlocking = false
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      isBlocking = isBlockingSystemModal(element, in: springboard)
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_MODAL_CHECK_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return false
+    }
+    return isBlocking
+  }
+
+  private func isBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool {
+    guard element.exists else { return false }
+    let frame = element.frame
+    if frame.isNull || frame.isEmpty { return false }
+
+    let viewport = springboard.frame
+    if viewport.isNull || viewport.isEmpty { return false }
+
+    let center = CGPoint(x: frame.midX, y: frame.midY)
+    if !viewport.contains(center) { return false }
+
+    return true
+  }
+
+  private func actionableElements(in element: XCUIElement) -> [XCUIElement] {
+    var seen = Set<String>()
+    var actions: [XCUIElement] = []
+    let descendants = safeElementsQuery {
+      element.descendants(matching: .any).allElementsBoundByIndex
+    }
+    for candidate in descendants {
+      if !safeIsActionableCandidate(candidate, seen: &seen) { continue }
+      actions.append(candidate)
+    }
+    return actions
+  }
+
+  private func safeIsActionableCandidate(_ candidate: XCUIElement, seen: inout Set<String>) -> Bool {
+    var include = false
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      if !candidate.exists || !candidate.isHittable { return }
+      if !actionableTypes.contains(candidate.elementType) { return }
+      let frame = candidate.frame
+      if frame.isNull || frame.isEmpty { return }
+      let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)"
+      if seen.contains(key) { return }
+      seen.insert(key)
+      include = true
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_MODAL_ACTION_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return false
+    }
+    return include
+  }
+
+  private func preferredSystemModalTitle(_ element: XCUIElement) -> String {
+    let label = element.label
+    if !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+      return label
+    }
+    let identifier = element.identifier
+    if !identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+      return identifier
+    }
+    return "System Alert"
+  }
+
+  private func makeSnapshotNode(
+    element: XCUIElement,
+    index: Int,
+    type: String,
+    labelOverride: String? = nil,
+    identifierOverride: String? = nil,
+    depth: Int,
+    hittableOverride: Bool? = nil
+  ) -> SnapshotNode {
+    let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
+    let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines)
+    return SnapshotNode(
+      index: index,
+      type: type,
+      label: label.isEmpty ? nil : label,
+      identifier: identifier.isEmpty ? nil : identifier,
+      value: nil,
+      rect: snapshotRect(from: element.frame),
+      enabled: element.isEnabled,
+      hittable: hittableOverride ?? element.isHittable,
+      depth: depth
+    )
+  }
+
+  private func safeMakeSnapshotNode(
+    element: XCUIElement,
+    index: Int,
+    type: String,
+    labelOverride: String? = nil,
+    identifierOverride: String? = nil,
+    depth: Int,
+    hittableOverride: Bool? = nil
+  ) -> SnapshotNode? {
+    var node: SnapshotNode?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      node = makeSnapshotNode(
+        element: element,
+        index: index,
+        type: type,
+        labelOverride: labelOverride,
+        identifierOverride: identifierOverride,
+        depth: depth,
+        hittableOverride: hittableOverride
+      )
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_MODAL_NODE_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return node
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -0,0 +1,124 @@
+import XCTest
+import Network
+
+extension RunnerTests {
+  // MARK: - Connection Lifecycle
+
+  func handle(connection: NWConnection) {
+    receiveRequest(connection: connection, buffer: Data())
+  }
+
+  // MARK: - Request Parsing
+
+  private func receiveRequest(connection: NWConnection, buffer: Data) {
+    connection.receive(minimumIncompleteLength: 1, maximumLength: 1024 * 1024) { [weak self] data, _, _, _ in
+      guard let self = self, let data = data else {
+        connection.cancel()
+        return
+      }
+      if buffer.count + data.count > self.maxRequestBytes {
+        let response = self.jsonResponse(
+          status: 413,
+          response: Response(ok: false, error: ErrorPayload(message: "request too large")),
+        )
+        connection.send(content: response, completion: .contentProcessed { [weak self] _ in
+          connection.cancel()
+          self?.finish()
+        })
+        return
+      }
+      let combined = buffer + data
+      if let body = self.parseRequest(data: combined) {
+        let result = self.handleRequestBody(body)
+        connection.send(content: result.data, completion: .contentProcessed { _ in
+          connection.cancel()
+          if result.shouldFinish {
+            self.finish()
+          }
+        })
+      } else {
+        self.receiveRequest(connection: connection, buffer: combined)
+      }
+    }
+  }
+
+  private func parseRequest(data: Data) -> Data? {
+    guard let headerEnd = data.range(of: Data("\r\n\r\n".utf8)) else {
+      return nil
+    }
+    let headerData = data.subdata(in: 0..<headerEnd.lowerBound)
+    let bodyStart = headerEnd.upperBound
+    let headers = String(decoding: headerData, as: UTF8.self)
+    let contentLength = extractContentLength(headers: headers)
+    guard let contentLength = contentLength else {
+      return nil
+    }
+    if data.count < bodyStart + contentLength {
+      return nil
+    }
+    let body = data.subdata(in: bodyStart..<(bodyStart + contentLength))
+    return body
+  }
+
+  private func extractContentLength(headers: String) -> Int? {
+    for line in headers.split(separator: "\r\n") {
+      let parts = line.split(separator: ":", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }
+      if parts.count == 2 && parts[0].lowercased() == "content-length" {
+        return Int(parts[1])
+      }
+    }
+    return nil
+  }
+
+  private func handleRequestBody(_ body: Data) -> (data: Data, shouldFinish: Bool) {
+    guard let json = String(data: body, encoding: .utf8) else {
+      return (
+        jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
+        false
+      )
+    }
+    guard let data = json.data(using: .utf8) else {
+      return (
+        jsonResponse(status: 400, response: Response(ok: false, error: ErrorPayload(message: "invalid json"))),
+        false
+      )
+    }
+
+    do {
+      let command = try JSONDecoder().decode(Command.self, from: data)
+      let response = try execute(command: command)
+      return (jsonResponse(status: 200, response: response), command.command == .shutdown)
+    } catch {
+      return (
+        jsonResponse(status: 500, response: Response(ok: false, error: ErrorPayload(message: "\(error)"))),
+        false
+      )
+    }
+  }
+
+  // MARK: - Response Encoding
+
+  private func jsonResponse(status: Int, response: Response) -> Data {
+    let encoder = JSONEncoder()
+    let body = (try? encoder.encode(response)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
+    return httpResponse(status: status, body: body)
+  }
+
+  private func httpResponse(status: Int, body: String) -> Data {
+    let headers = [
+      "HTTP/1.1 \(status) OK",
+      "Content-Type: application/json",
+      "Content-Length: \(body.utf8.count)",
+      "Connection: close",
+      "",
+      body,
+    ].joined(separator: "\r\n")
+    return Data(headers.utf8)
+  }
+
+  private func finish() {
+    listener?.cancel()
+    listener = nil
+    doneExpectation?.fulfill()
+  }
+}