agent-device 0.21.6 → 0.21.7

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift

@@ -1,4 +1,5 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 private enum KeyboardDismissObservationTiming {
   static let timeout: TimeInterval = 2
@@ -30,6 +31,164 @@ enum RunnerKeyboardDismissMechanism: String {
   case dismissKey
 }
 
+
+
+
+
+
+
+
+enum RunnerKeyboardBandFact: Equatable {
+  case visible(CGRect)
+  case absent
+  case unmeasurable(String)
+
+  var payload: KeyboardBandFactPayload {
+    switch self {
+    case .visible(let frame):
+      return KeyboardBandFactPayload(
+        kind: "visible",
+        frame: SnapshotRect(
+          x: Double(frame.origin.x),
+          y: Double(frame.origin.y),
+          width: Double(frame.size.width),
+          height: Double(frame.size.height)
+        ),
+        reason: nil
+      )
+    case .absent:
+      return KeyboardBandFactPayload(kind: "absent", frame: nil, reason: nil)
+    case .unmeasurable(let reason):
+      return KeyboardBandFactPayload(kind: "unmeasurable", frame: nil, reason: reason)
+    }
+  }
+
+  var logKind: String {
+    switch self {
+    case .visible: return "visible"
+    case .absent: return "absent"
+    case .unmeasurable: return "unmeasurable"
+    }
+  }
+
+  var logReason: String {
+    switch self {
+    case .visible, .absent: return "-"
+    case .unmeasurable(let reason): return reason
+    }
+  }
+}
+
+
+
+
+enum RunnerKeyboardBandReason {
+
+  static let queryFailed = "keyboard-frame-query-failed"
+
+
+  static let unusableFrame = "keyboard-frame-unusable"
+
+
+
+  static let queryTimeout = "keyboard-frame-query-timeout"
+
+  static let budgetExhausted = "capture-plan-budget-exhausted"
+}
+
+
+
+
+
+
+
+
+func runnerKeyboardFrameIsUsable(_ frame: CGRect) -> Bool {
+  let size = frame.size
+  let origin = frame.origin
+  return [origin.x, origin.y, size.width, size.height].allSatisfy(\.isFinite)
+    && size.width > 0 && size.height > 0
+}
+
+
+
+
+
+
+func runnerKeyboardBandFact(
+  readSucceeded: Bool,
+  exists: Bool,
+  frame: CGRect
+) -> RunnerKeyboardBandFact {
+  guard readSucceeded else { return .unmeasurable(RunnerKeyboardBandReason.queryFailed) }
+  guard exists else { return .absent }
+  guard runnerKeyboardFrameIsUsable(frame) else {
+    return .unmeasurable(RunnerKeyboardBandReason.unusableFrame)
+  }
+  return .visible(frame)
+}
+
+private struct KeyboardBandProbeTimeout: Error {}
+
+extension RunnerTests {
+
+
+
+
+
+
+
+
+
+
+
+  func captureKeyboardBandFact(app: XCUIApplication, deadline: Date) -> RunnerKeyboardBandFact? {
+#if os(iOS)
+    let slice = min(Self.keyboardBandProbeBudget, max(0, deadline.timeIntervalSinceNow))
+    guard slice > 0 else {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_KEYBOARD_BAND_FACT kind=unmeasurable reason=%@ elapsedMs=0",
+        RunnerKeyboardBandReason.budgetExhausted
+      )
+      return .unmeasurable(RunnerKeyboardBandReason.budgetExhausted)
+    }
+    let startedAt = Date()
+    let fact: RunnerKeyboardBandFact
+    do {
+      fact = try runMainThreadWork(
+        "keyboard_band",
+        timeout: slice,
+        timeoutError: { KeyboardBandProbeTimeout() }
+      ) {
+        self.keyboardBandFact(app: app)
+      }
+    } catch {
+      fact = .unmeasurable(RunnerKeyboardBandReason.queryTimeout)
+    }
+    NSLog(
+      "AGENT_DEVICE_RUNNER_KEYBOARD_BAND_FACT kind=%@ reason=%@ elapsedMs=%d",
+      fact.logKind,
+      fact.logReason,
+      Int(Date().timeIntervalSince(startedAt) * 1000)
+    )
+    return fact
+#else
+    return nil
+#endif
+  }
+
+  private func keyboardBandFact(app: XCUIApplication) -> RunnerKeyboardBandFact {
+    let read = safely("KEYBOARD_BAND_FACT") { () -> RunnerKeyboardBandFact? in
+      let keyboard = app.keyboards.firstMatch
+      guard keyboard.exists else {
+        return runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero)
+      }
+      return runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: keyboard.frame)
+    }
+    return read ?? .unmeasurable(RunnerKeyboardBandReason.queryFailed)
+  }
+}
+
 extension RunnerTests {
   func isKeyboardVisible(app: XCUIApplication) -> Bool {
     return visibleKeyboardFrame(app: app) != nil
@@ -312,6 +471,88 @@ func runnerScreenshotStabilitySettled(
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -200,6 +200,20 @@ extension RunnerTests {
     return true
   }
 
+
+
+
+
+
+
+  func soleOtherActiveApplicationPid(excluding sessionPid: Int?) -> Int? {
+    let pids = RunnerAXSnapshotBridge.activeApplicationProcessIdentifiers().compactMap {
+      ($0 as? NSNumber)?.intValue
+    }
+    let foreign = Set(pids.filter { $0 > 0 && $0 != sessionPid })
+    return foreign.count == 1 ? foreign.first : nil
+  }
+
   func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
     let target = XCUIApplication(bundleIdentifier: bundleId)
     let initialState = target.state
@@ -216,7 +230,22 @@ extension RunnerTests {
         bundleId
       )
     } else {
+
+
+      let otherActiveApplicationPid = soleOtherActiveApplicationPid(excluding: Self.processIdentifier(of: target))
       target.activate()
+      pendingTargetActivation = TargetActivationFactPayload(
+        reason: reason,
+        priorState: Int(initialState.rawValue),
+        otherActiveApplicationPid: otherActiveApplicationPid
+      )
+      NSLog(
+        "AGENT_DEVICE_RUNNER_ACTIVATE_FACT bundle=%@ reason=%@ priorState=%d otherActiveApplicationPid=%@",
+        bundleId,
+        reason,
+        initialState.rawValue,
+        otherActiveApplicationPid.map(String.init) ?? "-"
+      )
     }
     currentApp = target
     currentBundleId = bundleId

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -23,6 +23,7 @@ enum CommandType: String, Codable {
   case home
   case rotate
   case appSwitcher
+  case actionButton
   case keyboardDismiss
   case keyboardReturn
   case alert
@@ -97,6 +98,14 @@ extension CommandType {
     case .recordStop, .uptime, .terminate, .targetReset, .shutdown:
       return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
 
+
+
+
+
+
+    case .actionButton:
+      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
+
     case .status:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
 
@@ -239,6 +248,28 @@ extension Response {
     payload.runnerMainThreadBusy = value
     return Response(ok: ok, data: payload, error: error)
   }
+
+
+
+
+  func stampingTargetActivation(_ value: TargetActivationFactPayload) -> Response {
+    guard ok else { return self }
+    var payload = data ?? DataPayload()
+    payload.targetActivation = value
+    return Response(ok: ok, data: payload, error: error)
+  }
+}
+
+
+
+
+
+
+
+struct TargetActivationFactPayload: Codable {
+  let reason: String
+  let priorState: Int
+  let otherActiveApplicationPid: Int?
 }
 
 struct DataPayload: Codable {
@@ -253,6 +284,10 @@ struct DataPayload: Codable {
   var snapshotQuality: SnapshotQuality?
 
   var systemSurface: SystemSurfaceProvenancePayload?
+
+
+
+  var keyboard: KeyboardBandFactPayload?
   var gestureStartUptimeMs: Double?
   var gestureEndUptimeMs: Double?
   var x: Double?
@@ -294,6 +329,17 @@ struct DataPayload: Codable {
   var completedSteps: Int?
   var failedStepIndex: Int?
   var sequenceResults: [SequenceStepResult]?
+  var targetActivation: TargetActivationFactPayload?
+}
+
+
+
+
+
+struct KeyboardBandFactPayload: Codable, Equatable {
+  let kind: String
+  let frame: SnapshotRect?
+  let reason: String?
 }
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -23,33 +23,25 @@ extension RunnerTests {
 
 
 
-  func privateAXAcquisition(rawRoot: [String: Any], hint: CaptureHint, viewport: CGRect)
-    -> [RawAXNode]
-  {
+
+  func privateAXAcquisition(
+    rawRoot: [String: Any],
+    hint: CaptureHint
+  ) -> [RawAXNode] {
     var nodes: [RawAXNode] = []
-    appendPrivateAXNode(rawRoot, to: &nodes, hint: hint, viewport: viewport,
-      depth: 0, parentIndex: nil)
+    appendPrivateAXNode(rawRoot, to: &nodes, hint: hint, depth: 0, parentIndex: nil)
     return nodes
   }
 
   private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode],
-    hint: CaptureHint, viewport: CGRect, depth: Int, parentIndex: Int?)
+    hint: CaptureHint, depth: Int, parentIndex: Int?)
   {
     if let limit = hint.rawTraversalDepth, depth > limit { return }
     let fields = privateAXFields(raw)
     let index = nodes.count
-    nodes.append(
-      privateAXNode(
-        fields,
-        index: index,
-        depth: depth,
-        parentIndex: parentIndex,
-        viewport: viewport
-      )
-    )
+    nodes.append(privateAXNode(fields, index: index, depth: depth, parentIndex: parentIndex))
     for child in fields.children {
-      appendPrivateAXNode(child, to: &nodes, hint: hint, viewport: viewport,
-        depth: depth + 1, parentIndex: index)
+      appendPrivateAXNode(child, to: &nodes, hint: hint, depth: depth + 1, parentIndex: index)
     }
   }
 
@@ -70,21 +62,17 @@ extension RunnerTests {
     )
   }
 
-  private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int, parentIndex: Int?,
-    viewport: CGRect) -> RawAXNode
+  private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int,
+    parentIndex: Int?) -> RawAXNode
   {
-    RawAXNode(index: index,
+    return RawAXNode(index: index,
       type: fields.elementType.map(elementTypeName) ?? "Element(\(fields.rawType))",
       label: fields.label.isEmpty ? nil : fields.label,
       identifier: fields.identifier.isEmpty ? nil : fields.identifier,
       value: fields.value.isEmpty ? nil : fields.value,
-      rect: snapshotRect(from: fields.rect), enabled: fields.enabled,
+      rect: SnapshotRect(fields.rect), enabled: fields.enabled,
       focused: fields.focused, selected: fields.selected,
-      hittable: parentIndex != nil && SnapshotGeometry.isGeometricallyActionable(
-        enabled: fields.enabled,
-        frame: fields.rect,
-        viewport: viewport
-      ),
+      hittable: false,
       depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
       actions: fields.actions)
   }
@@ -219,6 +207,77 @@ extension RunnerTests {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+